From fca2a493251597967d5d758ea0748c66dd29371a Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Tue, 16 Apr 2024 21:46:57 -0700 Subject: [PATCH 001/862] [RISCV] Simplify FindRegWithEncoding in copyPhysRegVector. NFC (#89001) Instead of searching all encodings, we can convert the encoding back to a register and use getMatchingSuperReg. --- llvm/lib/Target/RISCV/RISCVInstrInfo.cpp | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp index 14b5cbea7172..8331fc0b8c30 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp @@ -361,15 +361,12 @@ void RISCVInstrInfo::copyPhysRegVector( return {RISCVII::LMUL_1, RISCV::VRRegClass, RISCV::VMV1R_V, RISCV::PseudoVMV_V_V_M1, RISCV::PseudoVMV_V_I_M1}; }; - auto FindRegWithEncoding = [&TRI](const TargetRegisterClass &RegClass, - uint16_t Encoding) { - ArrayRef Regs = RegClass.getRegisters(); - const auto *FoundReg = llvm::find_if(Regs, [&](MCPhysReg Reg) { - return TRI->getEncodingValue(Reg) == Encoding; - }); - // We should be always able to find one valid register. - assert(FoundReg != Regs.end()); - return *FoundReg; + auto FindRegWithEncoding = [TRI](const TargetRegisterClass &RegClass, + uint16_t Encoding) { + MCRegister Reg = RISCV::V0 + Encoding; + if (&RegClass == &RISCV::VRRegClass) + return Reg; + return TRI->getMatchingSuperReg(Reg, RISCV::sub_vrm1_0, &RegClass); }; while (I != NumRegs) { // For non-segment copying, we only do this once as the registers are always -- GitLab From a6fcbcce8f79adfb2e4338859f3a41fc2538bad1 Mon Sep 17 00:00:00 2001 From: Mark de Wever Date: Wed, 17 Apr 2024 07:59:43 +0200 Subject: [PATCH 002/862] [libc++][TZDB] Improves time zone format specifiers. (#85797) Per [tab:time.format.spec] %z The offset from UTC as specified in ISO 8601-1:2019, subclause 5.3.4.1. For example -0430 refers to 4 hours 30 minutes behind UTC. If the offset is zero, +0000 is used. The modified commands %Ez and %Oz insert a : between the hours and minutes: -04:30. If the offset information is not available, an exception of type format_error is thrown. Typically the modified versions Oz or Ez would have wording like The modified command %OS produces the locale's alternative representation. In this case the modified version does not depend on the locale. This change is a preparation for formatting sys_info which has time zone information. The function time_put<_CharT>::put() does not have proper time zone support, therefore it's a manual implementation. Fixes https://github.com/llvm/llvm-project/issues/78184 --- libcxx/include/__chrono/formatter.h | 50 ++++++++++++++++++- .../time.syn/formatter.file_time.pass.cpp | 39 ++------------- .../time/time.syn/formatter.sys_time.pass.cpp | 39 ++------------- 3 files changed, 56 insertions(+), 72 deletions(-) diff --git a/libcxx/include/__chrono/formatter.h b/libcxx/include/__chrono/formatter.h index b64cae529a29..d932a99f4b99 100644 --- a/libcxx/include/__chrono/formatter.h +++ b/libcxx/include/__chrono/formatter.h @@ -10,6 +10,7 @@ #ifndef _LIBCPP___CHRONO_FORMATTER_H #define _LIBCPP___CHRONO_FORMATTER_H +#include <__algorithm/ranges_copy.h> #include <__chrono/calendar.h> #include <__chrono/concepts.h> #include <__chrono/convert_to_tm.h> @@ -170,10 +171,45 @@ _LIBCPP_HIDE_FROM_ABI void __format_century(basic_stringstream<_CharT>& __sstr, __sstr << std::format(_LIBCPP_STATICALLY_WIDEN(_CharT, "{:02}"), __century); } +// Implements the %z format specifier according to [tab:time.format.spec], where +// '__modifier' signals %Oz or %Ez were used. (Both modifiers behave the same, +// so there is no need to distinguish between them.) +template +_LIBCPP_HIDE_FROM_ABI void +__format_zone_offset(basic_stringstream<_CharT>& __sstr, chrono::seconds __offset, bool __modifier) { + if (__offset < 0s) { + __sstr << _CharT('-'); + __offset = -__offset; + } else { + __sstr << _CharT('+'); + } + + chrono::hh_mm_ss __hms{__offset}; + std::ostreambuf_iterator<_CharT> __out_it{__sstr}; + if (__modifier) + std::format_to(__out_it, _LIBCPP_STATICALLY_WIDEN(_CharT, "{:%H:%M}"), __hms); + else + std::format_to(__out_it, _LIBCPP_STATICALLY_WIDEN(_CharT, "{:%H%M}"), __hms); +} + +// Helper to store the time zone information needed for formatting. +struct _LIBCPP_HIDE_FROM_ABI __time_zone { + // Typically these abbreviations are short and fit in the string's internal + // buffer. + string __abbrev; + chrono::seconds __offset; +}; + +template +_LIBCPP_HIDE_FROM_ABI __time_zone __convert_to_time_zone([[maybe_unused]] const _Tp& __value) { + return {"UTC", chrono::seconds{0}}; +} + template _LIBCPP_HIDE_FROM_ABI void __format_chrono_using_chrono_specs( basic_stringstream<_CharT>& __sstr, const _Tp& __value, basic_string_view<_CharT> __chrono_specs) { tm __t = std::__convert_to_tm(__value); + __time_zone __z = __formatter::__convert_to_time_zone(__value); const auto& __facet = std::use_facet>(__sstr.getloc()); for (auto __it = __chrono_specs.begin(); __it != __chrono_specs.end(); ++__it) { if (*__it == _CharT('%')) { @@ -296,9 +332,13 @@ _LIBCPP_HIDE_FROM_ABI void __format_chrono_using_chrono_specs( {__sstr}, __sstr, _CharT(' '), std::addressof(__t), std::to_address(__s), std::to_address(__it + 1)); } break; + case _CharT('z'): + __formatter::__format_zone_offset(__sstr, __z.__offset, false); + break; + case _CharT('Z'): - // TODO FMT Add proper timezone support. - __sstr << _LIBCPP_STATICALLY_WIDEN(_CharT, "UTC"); + // __abbrev is always a char so the copy may convert. + ranges::copy(__z.__abbrev, std::ostreambuf_iterator<_CharT>{__sstr}); break; case _CharT('O'): @@ -314,9 +354,15 @@ _LIBCPP_HIDE_FROM_ABI void __format_chrono_using_chrono_specs( break; } } + + // Oz produces the same output as Ez below. [[fallthrough]]; case _CharT('E'): ++__it; + if (*__it == 'z') { + __formatter::__format_zone_offset(__sstr, __z.__offset, true); + break; + } [[fallthrough]]; default: __facet.put( diff --git a/libcxx/test/std/time/time.syn/formatter.file_time.pass.cpp b/libcxx/test/std/time/time.syn/formatter.file_time.pass.cpp index b07282593d75..f57841cca862 100644 --- a/libcxx/test/std/time/time.syn/formatter.file_time.pass.cpp +++ b/libcxx/test/std/time/time.syn/formatter.file_time.pass.cpp @@ -904,12 +904,6 @@ static void test_valid_values_date_time() { template static void test_valid_values_time_zone() { -// The Apple CI gives %z='-0700' %Ez='-0700' %Oz='-0700' %Z='UTC' -// -0700 looks like the local time where the CI happens to reside, therefore -// omit this test on Apple. -// The Windows CI gives %z='-0000', but on local machines set to a different -// timezone, it gives e.g. %z='+0200'. -#if !defined(__APPLE__) && !defined(_WIN32) using namespace std::literals::chrono_literals; constexpr std::basic_string_view fmt = SV("{:%%z='%z'%t%%Ez='%Ez'%t%%Oz='%Oz'%t%%Z='%Z'%n}"); @@ -918,48 +912,23 @@ static void test_valid_values_time_zone() { const std::locale loc(LOCALE_ja_JP_UTF_8); std::locale::global(std::locale(LOCALE_fr_FR_UTF_8)); -# if defined(_AIX) // Non localized output using C-locale - check(SV("%z='UTC'\t%Ez='UTC'\t%Oz='UTC'\t%Z='UTC'\n"), + check(SV("%z='+0000'\t%Ez='+00:00'\t%Oz='+00:00'\t%Z='UTC'\n"), fmt, file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970 // Use the global locale (fr_FR) - check(SV("%z='UTC'\t%Ez='UTC'\t%Oz='UTC'\t%Z='UTC'\n"), + check(SV("%z='+0000'\t%Ez='+00:00'\t%Oz='+00:00'\t%Z='UTC'\n"), lfmt, file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970 - // Use supplied locale (ja_JP). This locale has a different alternate.a + // Use supplied locale (ja_JP). check(loc, - SV("%z='UTC'\t%Ez='UTC'\t%Oz='UTC'\t%Z='UTC'\n"), - lfmt, - file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970 -# else // defined(_AIX) - // Non localized output using C-locale - check(SV("%z='+0000'\t%Ez='+0000'\t%Oz='+0000'\t%Z='UTC'\n"), - fmt, - file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970 - - // Use the global locale (fr_FR) - check(SV("%z='+0000'\t%Ez='+0000'\t%Oz='+0000'\t%Z='UTC'\n"), + SV("%z='+0000'\t%Ez='+00:00'\t%Oz='+00:00'\t%Z='UTC'\n"), lfmt, file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970 - // Use supplied locale (ja_JP). This locale has a different alternate.a -# if defined(__FreeBSD__) - check(loc, - SV("%z='+0000'\t%Ez='+0000'\t%Oz='+0000'\t%Z='UTC'\n"), - lfmt, - file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970 -# else - check(loc, - SV("%z='+0000'\t%Ez='+0000'\t%Oz='+〇'\t%Z='UTC'\n"), - lfmt, - file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970 -# endif -# endif // defined(_AIX) std::locale::global(std::locale::classic()); -#endif // !defined(__APPLE__) && !defined(_WIN32) } template diff --git a/libcxx/test/std/time/time.syn/formatter.sys_time.pass.cpp b/libcxx/test/std/time/time.syn/formatter.sys_time.pass.cpp index 2fed270cbade..3a7d6f9a6b01 100644 --- a/libcxx/test/std/time/time.syn/formatter.sys_time.pass.cpp +++ b/libcxx/test/std/time/time.syn/formatter.sys_time.pass.cpp @@ -900,12 +900,6 @@ static void test_valid_values_date_time() { template static void test_valid_values_time_zone() { -// The Apple CI gives %z='-0700' %Ez='-0700' %Oz='-0700' %Z='UTC' -// -0700 looks like the local time where the CI happens to reside, therefore -// omit this test on Apple. -// The Windows CI gives %z='-0000', but on local machines set to a different -// timezone, it gives e.g. %z='+0200'. -#if !defined(__APPLE__) && !defined(_WIN32) using namespace std::literals::chrono_literals; constexpr std::basic_string_view fmt = SV("{:%%z='%z'%t%%Ez='%Ez'%t%%Oz='%Oz'%t%%Z='%Z'%n}"); @@ -914,48 +908,23 @@ static void test_valid_values_time_zone() { const std::locale loc(LOCALE_ja_JP_UTF_8); std::locale::global(std::locale(LOCALE_fr_FR_UTF_8)); -# if defined(_AIX) // Non localized output using C-locale - check(SV("%z='UTC'\t%Ez='UTC'\t%Oz='UTC'\t%Z='UTC'\n"), + check(SV("%z='+0000'\t%Ez='+00:00'\t%Oz='+00:00'\t%Z='UTC'\n"), fmt, std::chrono::sys_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970 // Use the global locale (fr_FR) - check(SV("%z='UTC'\t%Ez='UTC'\t%Oz='UTC'\t%Z='UTC'\n"), + check(SV("%z='+0000'\t%Ez='+00:00'\t%Oz='+00:00'\t%Z='UTC'\n"), lfmt, std::chrono::sys_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970 - // Use supplied locale (ja_JP). This locale has a different alternate.a + // Use supplied locale (ja_JP). check(loc, - SV("%z='UTC'\t%Ez='UTC'\t%Oz='UTC'\t%Z='UTC'\n"), - lfmt, - std::chrono::sys_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970 -# else // defined(_AIX) - // Non localized output using C-locale - check(SV("%z='+0000'\t%Ez='+0000'\t%Oz='+0000'\t%Z='UTC'\n"), - fmt, - std::chrono::sys_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970 - - // Use the global locale (fr_FR) - check(SV("%z='+0000'\t%Ez='+0000'\t%Oz='+0000'\t%Z='UTC'\n"), + SV("%z='+0000'\t%Ez='+00:00'\t%Oz='+00:00'\t%Z='UTC'\n"), lfmt, std::chrono::sys_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970 - // Use supplied locale (ja_JP). This locale has a different alternate.a -# if defined(__FreeBSD__) - check(loc, - SV("%z='+0000'\t%Ez='+0000'\t%Oz='+0000'\t%Z='UTC'\n"), - lfmt, - std::chrono::sys_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970 -# else - check(loc, - SV("%z='+0000'\t%Ez='+0000'\t%Oz='+〇'\t%Z='UTC'\n"), - lfmt, - std::chrono::sys_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970 -# endif -# endif // defined(_AIX) std::locale::global(std::locale::classic()); -#endif // !defined(__APPLE__) && !defined(_WIN32) } template -- GitLab From e096c144921daba59963f15e89d2ca6fb32d3a78 Mon Sep 17 00:00:00 2001 From: Balazs Benics Date: Wed, 17 Apr 2024 08:02:49 +0200 Subject: [PATCH 003/862] [analyzer] Fix a security.cert.env.InvalidPtr crash Fixes #88181 --- clang/docs/ReleaseNotes.rst | 2 ++ .../StaticAnalyzer/Checkers/cert/InvalidPtrChecker.cpp | 6 +++++- clang/test/Analysis/invalid-ptr-checker.cpp | 10 ++++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 clang/test/Analysis/invalid-ptr-checker.cpp diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index efc32212f300..6099f8ab02f4 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -679,6 +679,8 @@ Static Analyzer but not under any case blocks if ``unroll-loops=true`` analyzer config is set. (#GH68819) - Support C++23 static operator calls. (#GH84972) +- Fixed a crash in ``security.cert.env.InvalidPtr`` checker when accidentally + matched user-defined ``strerror`` and similar library functions. (GH#88181) New features ^^^^^^^^^^^^ diff --git a/clang/lib/StaticAnalyzer/Checkers/cert/InvalidPtrChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/cert/InvalidPtrChecker.cpp index e5dd907c660d..b2947f590c4e 100644 --- a/clang/lib/StaticAnalyzer/Checkers/cert/InvalidPtrChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/cert/InvalidPtrChecker.cpp @@ -205,8 +205,12 @@ void InvalidPtrChecker::postPreviousReturnInvalidatingCall( CE, LCtx, CE->getType(), C.blockCount()); State = State->BindExpr(CE, LCtx, RetVal); + const auto *SymRegOfRetVal = + dyn_cast_or_null(RetVal.getAsRegion()); + if (!SymRegOfRetVal) + return; + // Remember to this region. - const auto *SymRegOfRetVal = cast(RetVal.getAsRegion()); const MemRegion *MR = SymRegOfRetVal->getBaseRegion(); State = State->set(FD, MR); diff --git a/clang/test/Analysis/invalid-ptr-checker.cpp b/clang/test/Analysis/invalid-ptr-checker.cpp new file mode 100644 index 000000000000..58bb45e0fb84 --- /dev/null +++ b/clang/test/Analysis/invalid-ptr-checker.cpp @@ -0,0 +1,10 @@ +// RUN: %clang_analyze_cc1 -analyzer-checker=core,security.cert.env.InvalidPtr -verify %s + +// expected-no-diagnostics + +namespace other { +int strerror(int errnum); // custom strerror +void no_crash_on_custom_strerror() { + (void)strerror(0); // no-crash +} +} // namespace other -- GitLab From 024281d4d26344f9613b9115ea1fcbdbdba23235 Mon Sep 17 00:00:00 2001 From: Balazs Benics Date: Wed, 17 Apr 2024 08:02:49 +0200 Subject: [PATCH 004/862] [analyzer] Harden security.cert.env.InvalidPtr checker fn matching Relates to #88181 --- .../Checkers/cert/InvalidPtrChecker.cpp | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/clang/lib/StaticAnalyzer/Checkers/cert/InvalidPtrChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/cert/InvalidPtrChecker.cpp index b2947f590c4e..fefe846b6911 100644 --- a/clang/lib/StaticAnalyzer/Checkers/cert/InvalidPtrChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/cert/InvalidPtrChecker.cpp @@ -48,14 +48,19 @@ private: bool InvalidatingGetEnv = false; // GetEnv can be treated invalidating and non-invalidating as well. - const CallDescription GetEnvCall{{"getenv"}, 1}; + const CallDescription GetEnvCall{CDM::CLibrary, {"getenv"}, 1}; const CallDescriptionMap EnvpInvalidatingFunctions = { - {{{"setenv"}, 3}, &InvalidPtrChecker::EnvpInvalidatingCall}, - {{{"unsetenv"}, 1}, &InvalidPtrChecker::EnvpInvalidatingCall}, - {{{"putenv"}, 1}, &InvalidPtrChecker::EnvpInvalidatingCall}, - {{{"_putenv_s"}, 2}, &InvalidPtrChecker::EnvpInvalidatingCall}, - {{{"_wputenv_s"}, 2}, &InvalidPtrChecker::EnvpInvalidatingCall}, + {{CDM::CLibrary, {"setenv"}, 3}, + &InvalidPtrChecker::EnvpInvalidatingCall}, + {{CDM::CLibrary, {"unsetenv"}, 1}, + &InvalidPtrChecker::EnvpInvalidatingCall}, + {{CDM::CLibrary, {"putenv"}, 1}, + &InvalidPtrChecker::EnvpInvalidatingCall}, + {{CDM::CLibrary, {"_putenv_s"}, 2}, + &InvalidPtrChecker::EnvpInvalidatingCall}, + {{CDM::CLibrary, {"_wputenv_s"}, 2}, + &InvalidPtrChecker::EnvpInvalidatingCall}, }; void postPreviousReturnInvalidatingCall(const CallEvent &Call, @@ -63,13 +68,13 @@ private: // SEI CERT ENV34-C const CallDescriptionMap PreviousCallInvalidatingFunctions = { - {{{"setlocale"}, 2}, + {{CDM::CLibrary, {"setlocale"}, 2}, &InvalidPtrChecker::postPreviousReturnInvalidatingCall}, - {{{"strerror"}, 1}, + {{CDM::CLibrary, {"strerror"}, 1}, &InvalidPtrChecker::postPreviousReturnInvalidatingCall}, - {{{"localeconv"}, 0}, + {{CDM::CLibrary, {"localeconv"}, 0}, &InvalidPtrChecker::postPreviousReturnInvalidatingCall}, - {{{"asctime"}, 1}, + {{CDM::CLibrary, {"asctime"}, 1}, &InvalidPtrChecker::postPreviousReturnInvalidatingCall}, }; -- GitLab From b851c7f1fc4fd83ea84d565bbdc30fd0d356788c Mon Sep 17 00:00:00 2001 From: martinboehme Date: Wed, 17 Apr 2024 08:05:43 +0200 Subject: [PATCH 005/862] [clang][dataflow] Support `StmtExpr` in `PropagateResultObject()`. (#88872) This patch adds a test that assert-fails without the fix. --- .../FlowSensitive/DataflowEnvironment.cpp | 5 ++++ .../Analysis/FlowSensitive/TransferTest.cpp | 26 +++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp index 3bf3807268be..f2b4a67e5bc9 100644 --- a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp +++ b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp @@ -470,6 +470,11 @@ public: return; } + if (auto *SE = dyn_cast(E)) { + PropagateResultObject(cast(SE->getSubStmt()->body_back()), Loc); + return; + } + // All other expression nodes that propagate a record prvalue should have // exactly one child. SmallVector Children(E->child_begin(), E->child_end()); diff --git a/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp b/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp index d8bcc3da4b8b..d7a51b009712 100644 --- a/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp +++ b/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp @@ -3182,6 +3182,32 @@ TEST(TransferTest, ResultObjectLocationForStdInitializerListExpr) { }); } +TEST(TransferTest, ResultObjectLocationForStmtExpr) { + std::string Code = R"( + struct S {}; + void target() { + S s = ({ S(); }); + // [[p]] + } + )"; + using ast_matchers::cxxConstructExpr; + using ast_matchers::match; + using ast_matchers::selectFirst; + using ast_matchers::traverse; + runDataflow( + Code, + [](const llvm::StringMap> &Results, + ASTContext &ASTCtx) { + const Environment &Env = getEnvironmentAtAnnotation(Results, "p"); + + auto *Construct = selectFirst( + "construct", match(cxxConstructExpr().bind("construct"), ASTCtx)); + + EXPECT_EQ(&Env.getResultObjectLocation(*Construct), + &getLocForDecl(ASTCtx, Env, "s")); + }); +} + TEST(TransferTest, ResultObjectLocationPropagatesThroughConditionalOperator) { std::string Code = R"( struct A { -- GitLab From 47148832d4e3bf4901430732f1af6673147accb2 Mon Sep 17 00:00:00 2001 From: Hideto Ueno Date: Wed, 17 Apr 2024 15:09:47 +0900 Subject: [PATCH 006/862] [mlir][python] Add `walk` method to PyOperationBase (#87962) This commit adds `walk` method to PyOperationBase that uses a python object as a callback, e.g. `op.walk(callback)`. Currently callback must return a walk result explicitly. We(SiFive) have implemented walk method with python in our internal python tool for a while. However the overhead of python is expensive and it didn't scale well for large MLIR files. Just replacing walk with this version reduced the entire execution time of the tool by 30~40% and there are a few configs that the tool takes several hours to finish so this commit significantly improves tool performance. --- mlir/include/mlir-c/IR.h | 10 ++- .../mlir/Bindings/Python/PybindAdaptors.h | 1 + mlir/lib/Bindings/Python/IRCore.cpp | 32 +++++++- mlir/lib/Bindings/Python/IRModule.h | 4 + mlir/lib/CAPI/IR/IR.cpp | 21 +++++- mlir/test/CAPI/ir.c | 58 +++++++++++--- mlir/test/python/ir/operation.py | 75 +++++++++++++++++++ 7 files changed, 184 insertions(+), 17 deletions(-) diff --git a/mlir/include/mlir-c/IR.h b/mlir/include/mlir-c/IR.h index 82da511f807a..32abacf35313 100644 --- a/mlir/include/mlir-c/IR.h +++ b/mlir/include/mlir-c/IR.h @@ -705,6 +705,13 @@ MLIR_CAPI_EXPORTED void mlirOperationMoveAfter(MlirOperation op, MLIR_CAPI_EXPORTED void mlirOperationMoveBefore(MlirOperation op, MlirOperation other); +/// Operation walk result. +typedef enum MlirWalkResult { + MlirWalkResultAdvance, + MlirWalkResultInterrupt, + MlirWalkResultSkip +} MlirWalkResult; + /// Traversal order for operation walk. typedef enum MlirWalkOrder { MlirWalkPreOrder, @@ -713,7 +720,8 @@ typedef enum MlirWalkOrder { /// Operation walker type. The handler is passed an (opaque) reference to an /// operation and a pointer to a `userData`. -typedef void (*MlirOperationWalkCallback)(MlirOperation, void *userData); +typedef MlirWalkResult (*MlirOperationWalkCallback)(MlirOperation, + void *userData); /// Walks operation `op` in `walkOrder` and calls `callback` on that operation. /// `*userData` is passed to the callback as well and can be used to tunnel some diff --git a/mlir/include/mlir/Bindings/Python/PybindAdaptors.h b/mlir/include/mlir/Bindings/Python/PybindAdaptors.h index 52f632125191..d8f22c7aa170 100644 --- a/mlir/include/mlir/Bindings/Python/PybindAdaptors.h +++ b/mlir/include/mlir/Bindings/Python/PybindAdaptors.h @@ -18,6 +18,7 @@ #ifndef MLIR_BINDINGS_PYTHON_PYBINDADAPTORS_H #define MLIR_BINDINGS_PYTHON_PYBINDADAPTORS_H +#include #include #include #include diff --git a/mlir/lib/Bindings/Python/IRCore.cpp b/mlir/lib/Bindings/Python/IRCore.cpp index 734f2f7f3f94..d875f4eba2b1 100644 --- a/mlir/lib/Bindings/Python/IRCore.cpp +++ b/mlir/lib/Bindings/Python/IRCore.cpp @@ -674,6 +674,7 @@ void PyMlirContext::clearOperationsInside(PyOperationBase &op) { data->rootOp.getOperation().getContext()->clearOperation(op); else data->rootSeen = true; + return MlirWalkResult::MlirWalkResultAdvance; }; mlirOperationWalk(op.getOperation(), invalidatingCallback, static_cast(&data), MlirWalkPreOrder); @@ -1249,6 +1250,21 @@ void PyOperationBase::writeBytecode(const py::object &fileObject, .str()); } +void PyOperationBase::walk( + std::function callback, + MlirWalkOrder walkOrder) { + PyOperation &operation = getOperation(); + operation.checkValid(); + MlirOperationWalkCallback walkCallback = [](MlirOperation op, + void *userData) { + auto *fn = + static_cast *>(userData); + return (*fn)(op); + }; + + mlirOperationWalk(operation, walkCallback, &callback, walkOrder); +} + py::object PyOperationBase::getAsm(bool binary, std::optional largeElementsLimit, bool enableDebugInfo, bool prettyDebugInfo, @@ -2511,6 +2527,15 @@ void mlir::python::populateIRCore(py::module &m) { .value("NOTE", MlirDiagnosticNote) .value("REMARK", MlirDiagnosticRemark); + py::enum_(m, "WalkOrder", py::module_local()) + .value("PRE_ORDER", MlirWalkPreOrder) + .value("POST_ORDER", MlirWalkPostOrder); + + py::enum_(m, "WalkResult", py::module_local()) + .value("ADVANCE", MlirWalkResultAdvance) + .value("INTERRUPT", MlirWalkResultInterrupt) + .value("SKIP", MlirWalkResultSkip); + //---------------------------------------------------------------------------- // Mapping of Diagnostics. //---------------------------------------------------------------------------- @@ -2989,8 +3014,7 @@ void mlir::python::populateIRCore(py::module &m) { py::arg("binary") = false, kOperationPrintStateDocstring) .def("print", py::overload_cast, bool, bool, bool, bool, - bool, py::object, bool>( - &PyOperationBase::print), + bool, py::object, bool>(&PyOperationBase::print), // Careful: Lots of arguments must match up with print method. py::arg("large_elements_limit") = py::none(), py::arg("enable_debug_info") = false, @@ -3038,7 +3062,9 @@ void mlir::python::populateIRCore(py::module &m) { return operation.createOpView(); }, "Detaches the operation from its parent block.") - .def("erase", [](PyOperationBase &self) { self.getOperation().erase(); }); + .def("erase", [](PyOperationBase &self) { self.getOperation().erase(); }) + .def("walk", &PyOperationBase::walk, py::arg("callback"), + py::arg("walk_order") = MlirWalkPostOrder); py::class_(m, "Operation", py::module_local()) .def_static("create", &PyOperation::create, py::arg("name"), diff --git a/mlir/lib/Bindings/Python/IRModule.h b/mlir/lib/Bindings/Python/IRModule.h index 9acfdde25ae0..b038a0c54d29 100644 --- a/mlir/lib/Bindings/Python/IRModule.h +++ b/mlir/lib/Bindings/Python/IRModule.h @@ -579,6 +579,10 @@ public: void writeBytecode(const pybind11::object &fileObject, std::optional bytecodeVersion); + // Implement the walk method. + void walk(std::function callback, + MlirWalkOrder walkOrder); + /// Moves the operation before or after the other operation. void moveAfter(PyOperationBase &other); void moveBefore(PyOperationBase &other); diff --git a/mlir/lib/CAPI/IR/IR.cpp b/mlir/lib/CAPI/IR/IR.cpp index cdb64f4ec4a4..a72cd247e73f 100644 --- a/mlir/lib/CAPI/IR/IR.cpp +++ b/mlir/lib/CAPI/IR/IR.cpp @@ -717,17 +717,34 @@ void mlirOperationMoveBefore(MlirOperation op, MlirOperation other) { return unwrap(op)->moveBefore(unwrap(other)); } +static mlir::WalkResult unwrap(MlirWalkResult result) { + switch (result) { + case MlirWalkResultAdvance: + return mlir::WalkResult::advance(); + + case MlirWalkResultInterrupt: + return mlir::WalkResult::interrupt(); + + case MlirWalkResultSkip: + return mlir::WalkResult::skip(); + } +} + void mlirOperationWalk(MlirOperation op, MlirOperationWalkCallback callback, void *userData, MlirWalkOrder walkOrder) { switch (walkOrder) { case MlirWalkPreOrder: unwrap(op)->walk( - [callback, userData](Operation *op) { callback(wrap(op), userData); }); + [callback, userData](Operation *op) { + return unwrap(callback(wrap(op), userData)); + }); break; case MlirWalkPostOrder: unwrap(op)->walk( - [callback, userData](Operation *op) { callback(wrap(op), userData); }); + [callback, userData](Operation *op) { + return unwrap(callback(wrap(op), userData)); + }); } } diff --git a/mlir/test/CAPI/ir.c b/mlir/test/CAPI/ir.c index 8e79338c57a2..3d05b2a12dd8 100644 --- a/mlir/test/CAPI/ir.c +++ b/mlir/test/CAPI/ir.c @@ -2244,9 +2244,22 @@ typedef struct { const char *x; } callBackData; -void walkCallBack(MlirOperation op, void *rootOpVoid) { +MlirWalkResult walkCallBack(MlirOperation op, void *rootOpVoid) { fprintf(stderr, "%s: %s\n", ((callBackData *)(rootOpVoid))->x, mlirIdentifierStr(mlirOperationGetName(op)).data); + return MlirWalkResultAdvance; +} + +MlirWalkResult walkCallBackTestWalkResult(MlirOperation op, void *rootOpVoid) { + fprintf(stderr, "%s: %s\n", ((callBackData *)(rootOpVoid))->x, + mlirIdentifierStr(mlirOperationGetName(op)).data); + if (strcmp(mlirIdentifierStr(mlirOperationGetName(op)).data, "func.func") == + 0) + return MlirWalkResultSkip; + if (strcmp(mlirIdentifierStr(mlirOperationGetName(op)).data, "arith.addi") == + 0) + return MlirWalkResultInterrupt; + return MlirWalkResultAdvance; } int testOperationWalk(MlirContext ctx) { @@ -2259,6 +2272,9 @@ int testOperationWalk(MlirContext ctx) { " arith.addi %1, %1: i32\n" " return\n" " }\n" + " func.func @bar() {\n" + " return\n" + " }\n" "}"; MlirModule module = mlirModuleCreateParse(ctx, mlirStringRefCreateFromCString(moduleString)); @@ -2266,22 +2282,42 @@ int testOperationWalk(MlirContext ctx) { callBackData data; data.x = "i love you"; - // CHECK: i love you: arith.constant - // CHECK: i love you: arith.addi - // CHECK: i love you: func.return - // CHECK: i love you: func.func - // CHECK: i love you: builtin.module + // CHECK-NEXT: i love you: arith.constant + // CHECK-NEXT: i love you: arith.addi + // CHECK-NEXT: i love you: func.return + // CHECK-NEXT: i love you: func.func + // CHECK-NEXT: i love you: func.return + // CHECK-NEXT: i love you: func.func + // CHECK-NEXT: i love you: builtin.module mlirOperationWalk(mlirModuleGetOperation(module), walkCallBack, (void *)(&data), MlirWalkPostOrder); data.x = "i don't love you"; - // CHECK: i don't love you: builtin.module - // CHECK: i don't love you: func.func - // CHECK: i don't love you: arith.constant - // CHECK: i don't love you: arith.addi - // CHECK: i don't love you: func.return + // CHECK-NEXT: i don't love you: builtin.module + // CHECK-NEXT: i don't love you: func.func + // CHECK-NEXT: i don't love you: arith.constant + // CHECK-NEXT: i don't love you: arith.addi + // CHECK-NEXT: i don't love you: func.return + // CHECK-NEXT: i don't love you: func.func + // CHECK-NEXT: i don't love you: func.return mlirOperationWalk(mlirModuleGetOperation(module), walkCallBack, (void *)(&data), MlirWalkPreOrder); + + data.x = "interrupt"; + // Interrupted at `arith.addi` + // CHECK-NEXT: interrupt: arith.constant + // CHECK-NEXT: interrupt: arith.addi + mlirOperationWalk(mlirModuleGetOperation(module), walkCallBackTestWalkResult, + (void *)(&data), MlirWalkPostOrder); + + data.x = "skip"; + // Skip at `func.func` + // CHECK-NEXT: skip: builtin.module + // CHECK-NEXT: skip: func.func + // CHECK-NEXT: skip: func.func + mlirOperationWalk(mlirModuleGetOperation(module), walkCallBackTestWalkResult, + (void *)(&data), MlirWalkPreOrder); + mlirModuleDestroy(module); return 0; } diff --git a/mlir/test/python/ir/operation.py b/mlir/test/python/ir/operation.py index 04f8a9936e31..9666e63bda1e 100644 --- a/mlir/test/python/ir/operation.py +++ b/mlir/test/python/ir/operation.py @@ -1015,3 +1015,78 @@ def testOperationParse(): print( f"op_with_source_name: {o.get_asm(enable_debug_info=True, use_local_scope=True)}" ) + + +# CHECK-LABEL: TEST: testOpWalk +@run +def testOpWalk(): + ctx = Context() + ctx.allow_unregistered_dialects = True + module = Module.parse( + r""" + builtin.module { + func.func @f() { + func.return + } + } + """, + ctx, + ) + + def callback(op): + print(op.name) + return WalkResult.ADVANCE + + # Test post-order walk (default). + # CHECK-NEXT: Post-order + # CHECK-NEXT: func.return + # CHECK-NEXT: func.func + # CHECK-NEXT: builtin.module + print("Post-order") + module.operation.walk(callback) + + # Test pre-order walk. + # CHECK-NEXT: Pre-order + # CHECK-NEXT: builtin.module + # CHECK-NEXT: func.fun + # CHECK-NEXT: func.return + print("Pre-order") + module.operation.walk(callback, WalkOrder.PRE_ORDER) + + # Test interrput. + # CHECK-NEXT: Interrupt post-order + # CHECK-NEXT: func.return + print("Interrupt post-order") + + def callback(op): + print(op.name) + return WalkResult.INTERRUPT + + module.operation.walk(callback) + + # Test skip. + # CHECK-NEXT: Skip pre-order + # CHECK-NEXT: builtin.module + print("Skip pre-order") + + def callback(op): + print(op.name) + return WalkResult.SKIP + + module.operation.walk(callback, WalkOrder.PRE_ORDER) + + # Test exception. + # CHECK: Exception + # CHECK-NEXT: func.return + # CHECK-NEXT: Exception raised + print("Exception") + + def callback(op): + print(op.name) + raise ValueError + return WalkResult.ADVANCE + + try: + module.operation.walk(callback) + except ValueError: + print("Exception raised") -- GitLab From 1bccbe1f49abc39b9f980cf3f1b171da5541d1a4 Mon Sep 17 00:00:00 2001 From: martinboehme Date: Wed, 17 Apr 2024 08:17:56 +0200 Subject: [PATCH 007/862] [clang][dataflow] Treat `BuiltinBitCastExpr` correctly in `PropagateResultObject()`. (#88875) This patch includes a test that assert-fails without the fix. --- .../FlowSensitive/DataflowEnvironment.cpp | 6 ++++- .../Analysis/FlowSensitive/TransferTest.cpp | 26 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp index f2b4a67e5bc9..3f1600d9ac5d 100644 --- a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp +++ b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp @@ -419,7 +419,11 @@ public: // below them can initialize the same object (or part of it). if (isa(E) || isa(E) || isa(E) || isa(E) || isa(E) || - isa(E)) { + isa(E) || + // We treat `BuiltinBitCastExpr` as an "original initializer" too as + // it may not even be casting from a record type -- and even if it is, + // the two objects are in general of unrelated type. + isa(E)) { return; } if (auto *Op = dyn_cast(E); diff --git a/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp b/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp index d7a51b009712..97ec32126c1d 100644 --- a/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp +++ b/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp @@ -3208,6 +3208,32 @@ TEST(TransferTest, ResultObjectLocationForStmtExpr) { }); } +TEST(TransferTest, ResultObjectLocationForBuiltinBitCastExpr) { + std::string Code = R"( + struct S { int i; }; + void target(int i) { + S s = __builtin_bit_cast(S, i); + // [[p]] + } + )"; + using ast_matchers::explicitCastExpr; + using ast_matchers::match; + using ast_matchers::selectFirst; + using ast_matchers::traverse; + runDataflow( + Code, + [](const llvm::StringMap> &Results, + ASTContext &ASTCtx) { + const Environment &Env = getEnvironmentAtAnnotation(Results, "p"); + + auto *BuiltinBitCast = selectFirst( + "cast", match(explicitCastExpr().bind("cast"), ASTCtx)); + + EXPECT_EQ(&Env.getResultObjectLocation(*BuiltinBitCast), + &getLocForDecl(ASTCtx, Env, "s")); + }); +} + TEST(TransferTest, ResultObjectLocationPropagatesThroughConditionalOperator) { std::string Code = R"( struct A { -- GitLab From 64c649585ca23a0c996d8814d2796cd348441d69 Mon Sep 17 00:00:00 2001 From: Vlad Serebrennikov Date: Wed, 17 Apr 2024 09:20:55 +0300 Subject: [PATCH 008/862] [clang][NFC] Move `Sema::SkipBodyInfo` into namespace scope This makes it forward-declarable, and needed from splitting `Sema` up. --- clang/include/clang/Sema/Sema.h | 16 ++++++++-------- clang/lib/Parse/ParseDecl.cpp | 2 +- clang/lib/Parse/ParseDeclCXX.cpp | 2 +- clang/lib/Parse/ParseObjc.cpp | 4 ++-- clang/lib/Parse/Parser.cpp | 2 +- clang/lib/Sema/SemaDecl.cpp | 4 ++-- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h index 77150a318ee4..091c5c02f75d 100644 --- a/clang/include/clang/Sema/Sema.h +++ b/clang/include/clang/Sema/Sema.h @@ -352,6 +352,14 @@ private: llvm::function_ref ComputeType; }; +struct SkipBodyInfo { + SkipBodyInfo() = default; + bool ShouldSkip = false; + bool CheckSameAsPrevious = false; + NamedDecl *Previous = nullptr; + NamedDecl *New = nullptr; +}; + /// Describes the result of template argument deduction. /// /// The TemplateDeductionResult enumeration describes the result of @@ -2627,14 +2635,6 @@ public: return Entity->getOwningModule(); } - struct SkipBodyInfo { - SkipBodyInfo() = default; - bool ShouldSkip = false; - bool CheckSameAsPrevious = false; - NamedDecl *Previous = nullptr; - NamedDecl *New = nullptr; - }; - DeclGroupPtrTy ConvertDeclToDeclGroup(Decl *Ptr, Decl *OwnedType = nullptr); ParsedType getTypeName(const IdentifierInfo &II, SourceLocation NameLoc, diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp index c881b3750777..274ee7b10c17 100644 --- a/clang/lib/Parse/ParseDecl.cpp +++ b/clang/lib/Parse/ParseDecl.cpp @@ -5331,7 +5331,7 @@ void Parser::ParseEnumSpecifier(SourceLocation StartLoc, DeclSpec &DS, stripTypeAttributesOffDeclSpec(attrs, DS, TUK); - Sema::SkipBodyInfo SkipBody; + SkipBodyInfo SkipBody; if (!Name && TUK == Sema::TUK_Definition && Tok.is(tok::l_brace) && NextToken().is(tok::identifier)) SkipBody = Actions.shouldSkipAnonEnumBody(getCurScope(), diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp index cd4803d51bc1..51fd64b2d01a 100644 --- a/clang/lib/Parse/ParseDeclCXX.cpp +++ b/clang/lib/Parse/ParseDeclCXX.cpp @@ -2092,7 +2092,7 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind, TypeResult TypeResult = true; // invalid bool Owned = false; - Sema::SkipBodyInfo SkipBody; + SkipBodyInfo SkipBody; if (TemplateId) { // Explicit specialization, class template partial specialization, // or explicit instantiation. diff --git a/clang/lib/Parse/ParseObjc.cpp b/clang/lib/Parse/ParseObjc.cpp index 887d7a36cee7..671dcb71e51a 100644 --- a/clang/lib/Parse/ParseObjc.cpp +++ b/clang/lib/Parse/ParseObjc.cpp @@ -375,7 +375,7 @@ Decl *Parser::ParseObjCAtInterfaceDeclaration(SourceLocation AtLoc, Actions.ActOnTypedefedProtocols(protocols, protocolLocs, superClassId, superClassLoc); - Sema::SkipBodyInfo SkipBody; + SkipBodyInfo SkipBody; ObjCInterfaceDecl *ClsType = Actions.ActOnStartClassInterface( getCurScope(), AtLoc, nameId, nameLoc, typeParameterList, superClassId, superClassLoc, typeArgs, @@ -2133,7 +2133,7 @@ Parser::ParseObjCAtProtocolDeclaration(SourceLocation AtLoc, /*consumeLastToken=*/true)) return nullptr; - Sema::SkipBodyInfo SkipBody; + SkipBodyInfo SkipBody; ObjCProtocolDecl *ProtoType = Actions.ActOnStartProtocolInterface( AtLoc, protocolName, nameLoc, ProtocolRefs.data(), ProtocolRefs.size(), ProtocolLocs.data(), EndProtoLoc, attrs, &SkipBody); diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp index d6f2b9f448cd..ef46fc74cedc 100644 --- a/clang/lib/Parse/Parser.cpp +++ b/clang/lib/Parse/Parser.cpp @@ -1441,7 +1441,7 @@ Decl *Parser::ParseFunctionDefinition(ParsingDeclarator &D, // Tell the actions module that we have entered a function definition with the // specified Declarator for the function. - Sema::SkipBodyInfo SkipBody; + SkipBodyInfo SkipBody; Decl *Res = Actions.ActOnStartOfFunctionDef(getCurScope(), D, TemplateInfo.TemplateParams ? *TemplateInfo.TemplateParams diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp index 745cf41e204e..19abd5327b73 100644 --- a/clang/lib/Sema/SemaDecl.cpp +++ b/clang/lib/Sema/SemaDecl.cpp @@ -3037,7 +3037,7 @@ static void checkNewAttributesAfterDef(Sema &S, Decl *New, const Decl *Old) { if (isa(NewAttribute) || isa(NewAttribute)) { if (FunctionDecl *FD = dyn_cast(New)) { - Sema::SkipBodyInfo SkipBody; + SkipBodyInfo SkipBody; S.CheckForFunctionRedefinition(FD, cast(Def), &SkipBody); // If we're skipping this definition, drop the "alias" attribute. @@ -19999,7 +19999,7 @@ EnumConstantDecl *Sema::CheckEnumConstant(EnumDecl *Enum, Val, EnumVal); } -Sema::SkipBodyInfo Sema::shouldSkipAnonEnumBody(Scope *S, IdentifierInfo *II, +SkipBodyInfo Sema::shouldSkipAnonEnumBody(Scope *S, IdentifierInfo *II, SourceLocation IILoc) { if (!(getLangOpts().Modules || getLangOpts().ModulesLocalVisibility) || !getLangOpts().CPlusPlus) -- GitLab From 16f188761da1df6ba5e6627b8742aacfec8e9ec5 Mon Sep 17 00:00:00 2001 From: YunQiang Su Date: Wed, 17 Apr 2024 14:23:37 +0800 Subject: [PATCH 009/862] CompilerRT: Normalize COMPILER_RT_DEFAULT_TARGET_TRIPLE (#88835) If LLVM is configured with -DLLVM_DEFAULT_TARGET_TRIPLE, or compiler_rt is configured with -DCOMPILER_RT_DEFAULT_TARGET_TRIPLE, while the argument is not normalized, such as Debian-style vendor-less triple, clang will try to find libclang_rt in lib/, while libclang_rt is placed into lib/. Let's also place libclang_rt into lib/. --- compiler-rt/cmake/Modules/CompilerRTUtils.cmake | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/compiler-rt/cmake/Modules/CompilerRTUtils.cmake b/compiler-rt/cmake/Modules/CompilerRTUtils.cmake index e8e5f612d5b0..6d413f6753bc 100644 --- a/compiler-rt/cmake/Modules/CompilerRTUtils.cmake +++ b/compiler-rt/cmake/Modules/CompilerRTUtils.cmake @@ -368,6 +368,12 @@ macro(construct_compiler_rt_default_triple) "Default triple for which compiler-rt runtimes will be built.") endif() + if ("${CMAKE_C_COMPILER_ID}" MATCHES "Clang") + execute_process(COMMAND ${CMAKE_C_COMPILER} --target=${COMPILER_RT_DEFAULT_TARGET_TRIPLE} -print-effective-triple + OUTPUT_VARIABLE COMPILER_RT_DEFAULT_TARGET_TRIPLE + OUTPUT_STRIP_TRAILING_WHITESPACE) + endif() + string(REPLACE "-" ";" LLVM_TARGET_TRIPLE_LIST ${COMPILER_RT_DEFAULT_TARGET_TRIPLE}) list(GET LLVM_TARGET_TRIPLE_LIST 0 COMPILER_RT_DEFAULT_TARGET_ARCH) -- GitLab From b090569685699abe4a8031ad442a0f81e373146b Mon Sep 17 00:00:00 2001 From: Jesse Huang Date: Tue, 16 Apr 2024 23:36:27 -0700 Subject: [PATCH 010/862] [RISCV] Support Zama16b1p0 (#88474) This patch adds the support for Zama16b version 1.0, which has been added to RVA23U64 optional extensions recently --- clang/test/Preprocessor/riscv-target-features.c | 7 +++++++ llvm/docs/RISCVUsage.rst | 3 ++- llvm/lib/Support/RISCVISAInfo.cpp | 1 + llvm/lib/Target/RISCV/RISCVFeatures.td | 7 +++++++ llvm/test/CodeGen/RISCV/attributes.ll | 4 ++++ llvm/test/MC/RISCV/attribute-arch.s | 3 +++ llvm/unittests/Support/RISCVISAInfoTest.cpp | 1 + 7 files changed, 25 insertions(+), 1 deletion(-) diff --git a/clang/test/Preprocessor/riscv-target-features.c b/clang/test/Preprocessor/riscv-target-features.c index ec7764bb5381..646043681fe3 100644 --- a/clang/test/Preprocessor/riscv-target-features.c +++ b/clang/test/Preprocessor/riscv-target-features.c @@ -79,6 +79,7 @@ // CHECK-NOT: __riscv_za128rs {{.*$}} // CHECK-NOT: __riscv_za64rs {{.*$}} // CHECK-NOT: __riscv_zacas {{.*$}} +// CHECK-NOT: __riscv_zama16b {{.*$}} // CHECK-NOT: __riscv_zawrs {{.*$}} // CHECK-NOT: __riscv_zba {{.*$}} // CHECK-NOT: __riscv_zbb {{.*$}} @@ -704,6 +705,12 @@ // RUN: -o - | FileCheck --check-prefix=CHECK-ZACAS-EXT %s // CHECK-ZACAS-EXT: __riscv_zacas 1000000{{$}} +// RUN: %clang --target=riscv32 -march=rv32izama16b -x c -E -dM %s \ +// RUN: -o - | FileCheck --check-prefix=CHECK-ZAMA16B-EXT %s +// RUN: %clang --target=riscv64 -march=rv64izama16b -x c -E -dM %s \ +// RUN: -o - | FileCheck --check-prefix=CHECK-ZAMA16B-EXT %s +// CHECK-ZAMA16B-EXT: __riscv_zama16b 1000000{{$}} + // RUN: %clang --target=riscv32-unknown-linux-gnu \ // RUN: -march=rv32izawrs -E -dM %s \ // RUN: -o - | FileCheck --check-prefix=CHECK-ZAWRS-EXT %s diff --git a/llvm/docs/RISCVUsage.rst b/llvm/docs/RISCVUsage.rst index 6f5eba263def..a4cf17a8398a 100644 --- a/llvm/docs/RISCVUsage.rst +++ b/llvm/docs/RISCVUsage.rst @@ -119,6 +119,7 @@ on support follow. ``Za128rs`` Supported (`See note <#riscv-profiles-extensions-note>`__) ``Za64rs`` Supported (`See note <#riscv-profiles-extensions-note>`__) ``Zacas`` Supported (`See note <#riscv-zacas-note>`__) + ``Zama16b`` Supported (`See note <#riscv-profiles-extensions-note>`__) ``Zawrs`` Assembly Support ``Zba`` Supported ``Zbb`` Supported @@ -237,7 +238,7 @@ Supported .. _riscv-profiles-extensions-note: -``Za128rs``, ``Za64rs``, ``Zic64b``, ``Ziccamoa``, ``Ziccif``, ``Zicclsm``, ``Ziccrse``, ``Shcounterenvw``, ``Shgatpa``, ``Shtvala``, ``Shvsatpa``, ``Shvstvala``, ``Shvstvecd``, ``Ssccptr``, ``Sscounterenw``, ``Ssstateen``, ``Ssstrict``, ``Sstvala``, ``Sstvecd``, ``Ssu64xl``, ``Svade``, ``Svbare`` +``Za128rs``, ``Za64rs``, ``Zama16b``, ``Zic64b``, ``Ziccamoa``, ``Ziccif``, ``Zicclsm``, ``Ziccrse``, ``Shcounterenvw``, ``Shgatpa``, ``Shtvala``, ``Shvsatpa``, ``Shvstvala``, ``Shvstvecd``, ``Ssccptr``, ``Sscounterenw``, ``Ssstateen``, ``Ssstrict``, ``Sstvala``, ``Sstvecd``, ``Ssu64xl``, ``Svade``, ``Svbare`` These extensions are defined as part of the `RISC-V Profiles specification `__. They do not introduce any new features themselves, but instead describe existing hardware features. .. _riscv-zacas-note: diff --git a/llvm/lib/Support/RISCVISAInfo.cpp b/llvm/lib/Support/RISCVISAInfo.cpp index cbdc64bc7a97..fa967403ea44 100644 --- a/llvm/lib/Support/RISCVISAInfo.cpp +++ b/llvm/lib/Support/RISCVISAInfo.cpp @@ -119,6 +119,7 @@ static const RISCVSupportedExtension SupportedExtensions[] = { {"za128rs", {1, 0}}, {"za64rs", {1, 0}}, {"zacas", {1, 0}}, + {"zama16b", {1, 0}}, {"zawrs", {1, 0}}, {"zba", {1, 0}}, diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td index 561187c39a4a..f830ead5dd69 100644 --- a/llvm/lib/Target/RISCV/RISCVFeatures.td +++ b/llvm/lib/Target/RISCV/RISCVFeatures.td @@ -208,6 +208,13 @@ def HasStdExtAOrZalrsc "'A' (Atomic Instructions) or " "'Zalrsc' (Load-Reserved/Store-Conditional)">; +def FeatureStdExtZama16b + : SubtargetFeature<"zama16b", "HasStdExtZama16b", "true", + "'Zama16b' (Atomic 16-byte misaligned loads, stores and AMOs)">; +def HasStdExtZama16b : Predicate<"Subtarget->hasStdExtZama16b()">, + AssemblerPredicate<(all_of FeatureStdExtZama16b), + "'Zama16b' (Atomic 16-byte misaligned loads, stores and AMOs)">; + def FeatureStdExtZawrs : SubtargetFeature<"zawrs", "HasStdExtZawrs", "true", "'Zawrs' (Wait on Reservation Set)">; def HasStdExtZawrs : Predicate<"Subtarget->hasStdExtZawrs()">, diff --git a/llvm/test/CodeGen/RISCV/attributes.ll b/llvm/test/CodeGen/RISCV/attributes.ll index 2326599bf351..080783fdeec0 100644 --- a/llvm/test/CodeGen/RISCV/attributes.ll +++ b/llvm/test/CodeGen/RISCV/attributes.ll @@ -115,6 +115,7 @@ ; RUN: llc -mtriple=riscv32 -mattr=+zacas %s -o - | FileCheck --check-prefix=RV32ZACAS %s ; RUN: llc -mtriple=riscv32 -mattr=+experimental-zalasr %s -o - | FileCheck --check-prefix=RV32ZALASR %s ; RUN: llc -mtriple=riscv32 -mattr=+experimental-zalrsc %s -o - | FileCheck --check-prefix=RV32ZALRSC %s +; RUN: llc -mtriple=riscv32 -mattr=+zama16b %s -o - | FileCheck --check-prefixes=CHECK,RV32ZAMA16B %s ; RUN: llc -mtriple=riscv32 -mattr=+experimental-zicfilp %s -o - | FileCheck --check-prefix=RV32ZICFILP %s ; RUN: llc -mtriple=riscv32 -mattr=+experimental-zabha %s -o - | FileCheck --check-prefix=RV32ZABHA %s ; RUN: llc -mtriple=riscv32 -mattr=+experimental-ssnpm %s -o - | FileCheck --check-prefix=RV32SSNPM %s @@ -199,6 +200,7 @@ ; RUN: llc -mtriple=riscv64 -mattr=+xtheadvdot %s -o - | FileCheck --check-prefixes=CHECK,RV64XTHEADVDOT %s ; RUN: llc -mtriple=riscv64 -mattr=+za64rs %s -o - | FileCheck --check-prefixes=CHECK,RV64ZA64RS %s ; RUN: llc -mtriple=riscv64 -mattr=+za128rs %s -o - | FileCheck --check-prefixes=CHECK,RV64ZA128RS %s +; RUN: llc -mtriple=riscv64 -mattr=+zama16b %s -o - | FileCheck --check-prefixes=CHECK,RV64ZAMA16B %s ; RUN: llc -mtriple=riscv64 -mattr=+zawrs %s -o - | FileCheck --check-prefixes=CHECK,RV64ZAWRS %s ; RUN: llc -mtriple=riscv64 -mattr=+experimental-ztso %s -o - | FileCheck --check-prefixes=CHECK,RV64ZTSO %s ; RUN: llc -mtriple=riscv64 -mattr=+zca %s -o - | FileCheck --check-prefixes=CHECK,RV64ZCA %s @@ -370,6 +372,7 @@ ; RV32ZACAS: .attribute 5, "rv32i2p1_a2p1_zacas1p0" ; RV32ZALASR: .attribute 5, "rv32i2p1_zalasr0p1" ; RV32ZALRSC: .attribute 5, "rv32i2p1_zalrsc0p2" +; RV32ZAMA16B: .attribute 5, "rv32i2p1_zama16b1p0" ; RV32ZICFILP: .attribute 5, "rv32i2p1_zicfilp0p4" ; RV32ZABHA: .attribute 5, "rv32i2p1_a2p1_zabha1p0" ; RV32SSNPM: .attribute 5, "rv32i2p1_ssnpm0p8" @@ -418,6 +421,7 @@ ; RV64ZICBOZ: .attribute 5, "rv64i2p1_zicboz1p0" ; RV64ZA64RS: .attribute 5, "rv64i2p1_za64rs1p0" ; RV64ZA128RS: .attribute 5, "rv64i2p1_za128rs1p0" +; RV64ZAMA16B: .attribute 5, "rv64i2p1_zama16b1p0" ; RV64ZAWRS: .attribute 5, "rv64i2p1_zawrs1p0" ; RV64ZICBOP: .attribute 5, "rv64i2p1_zicbop1p0" ; RV64SHCOUNTERENW: .attribute 5, "rv64i2p1_shcounterenw1p0" diff --git a/llvm/test/MC/RISCV/attribute-arch.s b/llvm/test/MC/RISCV/attribute-arch.s index a8f493f781ec..8835ff22446c 100644 --- a/llvm/test/MC/RISCV/attribute-arch.s +++ b/llvm/test/MC/RISCV/attribute-arch.s @@ -270,6 +270,9 @@ .attribute arch, "rv32iza64rs1p0" # CHECK: attribute 5, "rv32i2p1_za64rs1p0" +.attribute arch, "rv32izama16b" +# CHECK: attribute 5, "rv32i2p1_zama16b1p0" + .attribute arch, "rv32izawrs1p0" # CHECK: attribute 5, "rv32i2p1_zawrs1p0" diff --git a/llvm/unittests/Support/RISCVISAInfoTest.cpp b/llvm/unittests/Support/RISCVISAInfoTest.cpp index 67012d2e6dc7..caf7bf0a3171 100644 --- a/llvm/unittests/Support/RISCVISAInfoTest.cpp +++ b/llvm/unittests/Support/RISCVISAInfoTest.cpp @@ -769,6 +769,7 @@ R"(All available -march extensions for RISC-V za128rs 1.0 za64rs 1.0 zacas 1.0 + zama16b 1.0 zawrs 1.0 zfa 1.0 zfh 1.0 -- GitLab From d35a64363bb851045387717d2ef7d6449b7b547f Mon Sep 17 00:00:00 2001 From: Mikhail Goncharov Date: Wed, 17 Apr 2024 08:33:07 +0200 Subject: [PATCH 011/862] Revert "Fix test from #83124 and #88902" This reverts commit 0a789ea8a829da345e46d8224d73b2ddaba6969f. Breaks builds, see discussion in https://github.com/llvm/llvm-project/pull/83124 --- clang/test/SemaCXX/PR41441.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/clang/test/SemaCXX/PR41441.cpp b/clang/test/SemaCXX/PR41441.cpp index d0f2917e52f2..0b012b33fce3 100644 --- a/clang/test/SemaCXX/PR41441.cpp +++ b/clang/test/SemaCXX/PR41441.cpp @@ -1,9 +1,6 @@ // RUN: %clang --target=x86_64-pc-linux -S -fno-discard-value-names -emit-llvm -o - %s | FileCheck %s -namespace std { - using size_t = decltype(sizeof(int)); -}; -void* operator new[](std::size_t, void*) noexcept; +#include // CHECK: call void @llvm.memset.p0.i64(ptr align 1 %x, i8 0, i64 8, i1 false) // CHECK: call void @llvm.memset.p0.i64(ptr align 16 %x, i8 0, i64 32, i1 false) -- GitLab From dbda478693104f78b142375862d66f3369ad8c78 Mon Sep 17 00:00:00 2001 From: Mikhail Goncharov Date: Wed, 17 Apr 2024 08:33:44 +0200 Subject: [PATCH 012/862] Revert "[Clang][Sema] placement new initializes typedef array with correct size (#88902)" This reverts commit 5c6af605b307213453a9a043532b9293db21b5c6. Breaks builds, see discussion in https://github.com/llvm/llvm-project/pull/83124 --- .../{PR41441.cpp => instantiate-new-placement-size.cpp} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename clang/test/SemaCXX/{PR41441.cpp => instantiate-new-placement-size.cpp} (75%) diff --git a/clang/test/SemaCXX/PR41441.cpp b/clang/test/SemaCXX/instantiate-new-placement-size.cpp similarity index 75% rename from clang/test/SemaCXX/PR41441.cpp rename to clang/test/SemaCXX/instantiate-new-placement-size.cpp index 0b012b33fce3..7a29d3dee849 100644 --- a/clang/test/SemaCXX/PR41441.cpp +++ b/clang/test/SemaCXX/instantiate-new-placement-size.cpp @@ -1,5 +1,5 @@ -// RUN: %clang --target=x86_64-pc-linux -S -fno-discard-value-names -emit-llvm -o - %s | FileCheck %s - +// RUN: %clang -S -fno-discard-value-names -emit-llvm -o - %s | FileCheck %s +// Issue no: 41441 #include // CHECK: call void @llvm.memset.p0.i64(ptr align 1 %x, i8 0, i64 8, i1 false) -- GitLab From dd84d23adc84cc0c3d2b8fb8f0c353279d99d27a Mon Sep 17 00:00:00 2001 From: Mikhail Goncharov Date: Wed, 17 Apr 2024 08:36:29 +0200 Subject: [PATCH 013/862] Revert "[Clang][Sema] placement new initializes typedef array with correct size (#83124)" This reverts commit c309dc6d0759b23b570c563f611530ff1a49e1bd. Breaks builds, see discussion in https://github.com/llvm/llvm-project/pull/83124 --- clang/docs/ReleaseNotes.rst | 1 - clang/lib/Sema/TreeTransform.h | 14 +------------ .../instantiate-new-placement-size.cpp | 20 ------------------- 3 files changed, 1 insertion(+), 34 deletions(-) delete mode 100644 clang/test/SemaCXX/instantiate-new-placement-size.cpp diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 6099f8ab02f4..96ad92b540b4 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -534,7 +534,6 @@ Bug Fixes to C++ Support Fixes (#GH70604), (#GH79754), (#GH84163), (#GH84425), (#GH86054), (#GH86398), and (#GH86399). - Fix a crash when deducing ``auto`` from an invalid dereference (#GH88329). - Fix a crash in requires expression with templated base class member function. Fixes (#GH84020). -- Placement new initializes typedef array with correct size (#GH41441) Bug Fixes to AST Handling ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h index 0c7fdb357235..eb05783a6219 100644 --- a/clang/lib/Sema/TreeTransform.h +++ b/clang/lib/Sema/TreeTransform.h @@ -12864,19 +12864,6 @@ TreeTransform::TransformCXXNewExpr(CXXNewExpr *E) { ArraySize = NewArraySize.get(); } - // Per C++0x [expr.new]p5, the type being constructed may be a - // typedef of an array type. - QualType AllocType = AllocTypeInfo->getType(); - if (ArraySize) { - if (const ConstantArrayType *Array = - SemaRef.Context.getAsConstantArrayType(AllocType)) { - ArraySize = IntegerLiteral::Create(SemaRef.Context, Array->getSize(), - SemaRef.Context.getSizeType(), - E->getBeginLoc()); - AllocType = Array->getElementType(); - } - } - // Transform the placement arguments (if any). bool ArgumentChanged = false; SmallVector PlacementArgs; @@ -12938,6 +12925,7 @@ TreeTransform::TransformCXXNewExpr(CXXNewExpr *E) { return E; } + QualType AllocType = AllocTypeInfo->getType(); if (!ArraySize) { // If no array size was specified, but the new expression was // instantiated with an array type (e.g., "new T" where T is diff --git a/clang/test/SemaCXX/instantiate-new-placement-size.cpp b/clang/test/SemaCXX/instantiate-new-placement-size.cpp deleted file mode 100644 index 7a29d3dee849..000000000000 --- a/clang/test/SemaCXX/instantiate-new-placement-size.cpp +++ /dev/null @@ -1,20 +0,0 @@ -// RUN: %clang -S -fno-discard-value-names -emit-llvm -o - %s | FileCheck %s -// Issue no: 41441 -#include - -// CHECK: call void @llvm.memset.p0.i64(ptr align 1 %x, i8 0, i64 8, i1 false) -// CHECK: call void @llvm.memset.p0.i64(ptr align 16 %x, i8 0, i64 32, i1 false) -template -void f() -{ - typedef TYPE TArray[8]; - - TArray x; - new(&x) TArray(); -} - -int main() -{ - f(); - f(); -} -- GitLab From bc3620d3a8b1be9534a5635431b0aa09cc50ff3c Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Wed, 17 Apr 2024 08:50:14 +0200 Subject: [PATCH 014/862] AMDGPU: Move libcall simplify into PeepholeEP (#88853) We were running this immediately on the incoming IR, which is still littered with temporary allocas obscuring trivial values. This needs to run after initial SROA to handle sincos insertion. --- .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 13 +++- .../amdgpu-libcall-sincos-pass-ordering.ll | 77 +++++++++++++++++++ .../AMDGPU/amdgpu-simplify-libcall-sincos.ll | 17 ++-- llvm/test/CodeGen/AMDGPU/simplify-libcalls.ll | 6 +- 4 files changed, 96 insertions(+), 17 deletions(-) create mode 100644 llvm/test/CodeGen/AMDGPU/amdgpu-libcall-sincos-pass-ordering.ll diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp index f7e552177d6f..305a6c8c3b92 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp @@ -655,9 +655,6 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks( PB.registerPipelineStartEPCallback( [](ModulePassManager &PM, OptimizationLevel Level) { FunctionPassManager FPM; - FPM.addPass(AMDGPUUseNativeCallsPass()); - if (EnableLibCallSimplify && Level != OptimizationLevel::O0) - FPM.addPass(AMDGPUSimplifyLibCallsPass()); PM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); if (EnableHipStdPar) PM.addPass(HipStdParAcceleratorCodeSelectionPass()); @@ -681,6 +678,16 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks( PM.addPass(AMDGPUAlwaysInlinePass()); }); + PB.registerPeepholeEPCallback( + [](FunctionPassManager &FPM, OptimizationLevel Level) { + if (Level == OptimizationLevel::O0) + return; + + FPM.addPass(AMDGPUUseNativeCallsPass()); + if (EnableLibCallSimplify) + FPM.addPass(AMDGPUSimplifyLibCallsPass()); + }); + PB.registerCGSCCOptimizerLateEPCallback( [this](CGSCCPassManager &PM, OptimizationLevel Level) { if (Level == OptimizationLevel::O0) diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-libcall-sincos-pass-ordering.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-libcall-sincos-pass-ordering.ll new file mode 100644 index 000000000000..6b835bb4eef6 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/amdgpu-libcall-sincos-pass-ordering.ll @@ -0,0 +1,77 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -O1 -amdgpu-prelink %s | FileCheck %s + +; Make sure that sin+cos -> sincos simplification happens after +; initial IR simplifications, otherwise we can't identify the common +; argument value. + +@.str = private unnamed_addr addrspace(4) constant [21 x i8] c"x: %f, y: %f, z: %f\0A\00", align 1 + +; Should have call to sincos declarations, not calls to the asm pseudo-libcalls +define protected amdgpu_kernel void @swdev456865(ptr addrspace(1) %out0, ptr addrspace(1) %out1, ptr addrspace(1) %out2, float noundef %x) #0 { +; CHECK-LABEL: define protected amdgpu_kernel void @swdev456865( +; CHECK-SAME: ptr addrspace(1) nocapture writeonly [[OUT0:%.*]], ptr addrspace(1) nocapture writeonly [[OUT1:%.*]], ptr addrspace(1) nocapture writeonly [[OUT2:%.*]], float noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[__SINCOS_:%.*]] = alloca float, align 4, addrspace(5) +; CHECK-NEXT: [[I_I:%.*]] = call float @_Z6sincosfPU3AS5f(float [[X]], ptr addrspace(5) [[__SINCOS_]]) #[[ATTR1:[0-9]+]] +; CHECK-NEXT: [[I_I2:%.*]] = load float, ptr addrspace(5) [[__SINCOS_]], align 4 +; CHECK-NEXT: [[ADD:%.*]] = fadd float [[I_I]], [[I_I2]] +; CHECK-NEXT: [[CONV:%.*]] = fpext float [[X]] to double +; CHECK-NEXT: [[CONV5:%.*]] = fpext float [[ADD]] to double +; CHECK-NEXT: store double [[CONV]], ptr addrspace(1) [[OUT0]], align 8 +; CHECK-NEXT: store double [[CONV5]], ptr addrspace(1) [[OUT1]], align 8 +; CHECK-NEXT: store double [[CONV5]], ptr addrspace(1) [[OUT2]], align 8 +; CHECK-NEXT: ret void +; +entry: + %x.addr = alloca float, align 4, addrspace(5) + %y = alloca float, align 4, addrspace(5) + %z = alloca float, align 4, addrspace(5) + store float %x, ptr addrspace(5) %x.addr, align 4 + call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) %y) + %i = load float, ptr addrspace(5) %x.addr, align 4 + %call = call float @_Z3sinf(float noundef %i) #3 + %i1 = load float, ptr addrspace(5) %x.addr, align 4 + %call1 = call float @_Z3cosf(float noundef %i1) #3 + %add = fadd float %call, %call1 + store float %add, ptr addrspace(5) %y, align 4 + call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) %z) + %i2 = load float, ptr addrspace(5) %x.addr, align 4 + %call2 = call float @_Z3cosf(float noundef %i2) #3 + %i3 = load float, ptr addrspace(5) %x.addr, align 4 + %call3 = call float @_Z3sinf(float noundef %i3) #3 + %add4 = fadd float %call2, %call3 + store float %add4, ptr addrspace(5) %z, align 4 + %i4 = load float, ptr addrspace(5) %x.addr, align 4 + %conv = fpext float %i4 to double + %i5 = load float, ptr addrspace(5) %y, align 4 + %conv5 = fpext float %i5 to double + %i6 = load float, ptr addrspace(5) %z, align 4 + %conv6 = fpext float %i6 to double + store double %conv, ptr addrspace(1) %out0, align 8 + store double %conv5, ptr addrspace(1) %out1, align 8 + store double %conv6, ptr addrspace(1) %out2, align 8 + call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) %z) + call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) %y) + ret void +} + +declare void @llvm.lifetime.start.p5(i64 immarg, ptr addrspace(5) nocapture) #1 +declare void @llvm.lifetime.end.p5(i64 immarg, ptr addrspace(5) nocapture) #1 + +define internal float @_Z3cosf(float noundef %arg) #2 { +bb: + %i = tail call float asm "pseudo-libcall-cos %0, %1", "=v,v"(float noundef %arg) #2 + ret float %i +} + +define internal float @_Z3sinf(float noundef %arg) #2 { +bb: + %i = tail call float asm "pseudo-libcall-sin %0, %1", "=v,v"(float noundef %arg) #2 + ret float %i +} + +attributes #0 = { norecurse nounwind } +attributes #1 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) } +attributes #2 = { mustprogress nofree norecurse nounwind willreturn memory(none) } +attributes #3 = { nounwind willreturn memory(none) } diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-sincos.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-sincos.ll index 5c56276eeb0f..9646d196da42 100644 --- a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-sincos.ll +++ b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-sincos.ll @@ -884,10 +884,9 @@ entry: define float @sincos_f32_unused_result_cos(float %x) { ; CHECK-LABEL: define float @sincos_f32_unused_result_cos -; CHECK-SAME: (float [[X:%.*]]) local_unnamed_addr #[[ATTR4]] { +; CHECK-SAME: (float [[X:%.*]]) local_unnamed_addr #[[ATTR5:[0-9]+]] { ; CHECK-NEXT: entry: -; CHECK-NEXT: [[__SINCOS_:%.*]] = alloca float, align 4, addrspace(5) -; CHECK-NEXT: [[TMP0:%.*]] = call contract float @_Z6sincosfPU3AS5f(float [[X]], ptr addrspace(5) [[__SINCOS_]]) +; CHECK-NEXT: [[TMP0:%.*]] = tail call contract float @_Z3sinf(float [[X]]) ; CHECK-NEXT: ret float [[TMP0]] ; entry: @@ -900,11 +899,9 @@ entry: define float @sincos_f32_unused_result_sin(float %x) { ; CHECK-LABEL: define float @sincos_f32_unused_result_sin -; CHECK-SAME: (float [[X:%.*]]) local_unnamed_addr #[[ATTR4]] { +; CHECK-SAME: (float [[X:%.*]]) local_unnamed_addr #[[ATTR5]] { ; CHECK-NEXT: entry: -; CHECK-NEXT: [[__SINCOS_:%.*]] = alloca float, align 4, addrspace(5) -; CHECK-NEXT: [[TMP0:%.*]] = call contract float @_Z6sincosfPU3AS5f(float [[X]], ptr addrspace(5) [[__SINCOS_]]) -; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(5) [[__SINCOS_]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = tail call contract float @_Z3cosf(float [[X]]) ; CHECK-NEXT: ret float [[TMP1]] ; entry: @@ -917,13 +914,11 @@ entry: define void @sincos_f32_repeated_uses(float %x, ptr addrspace(1) %sin_out, ptr addrspace(1) %cos_out) { ; CHECK-LABEL: define void @sincos_f32_repeated_uses -; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) [[SIN_OUT:%.*]], ptr addrspace(1) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR5:[0-9]+]] { +; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) [[SIN_OUT:%.*]], ptr addrspace(1) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR6:[0-9]+]] { ; CHECK-NEXT: entry: -; CHECK-NEXT: [[__SINCOS_:%.*]] = alloca float, align 4, addrspace(5) ; CHECK-NEXT: [[__SINCOS_3:%.*]] = alloca float, align 4, addrspace(5) ; CHECK-NEXT: [[TMP0:%.*]] = call contract float @_Z6sincosfPU3AS5f(float [[X]], ptr addrspace(5) [[__SINCOS_3]]) -; CHECK-NEXT: [[TMP1:%.*]] = call contract float @_Z6sincosfPU3AS5f(float [[X]], ptr addrspace(5) [[__SINCOS_]]) -; CHECK-NEXT: [[TMP2:%.*]] = load float, ptr addrspace(5) [[__SINCOS_]], align 4 +; CHECK-NEXT: [[TMP2:%.*]] = load float, ptr addrspace(5) [[__SINCOS_3]], align 4 ; CHECK-NEXT: store volatile float [[TMP0]], ptr addrspace(1) [[SIN_OUT]], align 4 ; CHECK-NEXT: store volatile float [[TMP0]], ptr addrspace(1) [[SIN_OUT]], align 4 ; CHECK-NEXT: store volatile float [[TMP2]], ptr addrspace(1) [[COS_OUT]], align 4 diff --git a/llvm/test/CodeGen/AMDGPU/simplify-libcalls.ll b/llvm/test/CodeGen/AMDGPU/simplify-libcalls.ll index 731a88278e51..204c8140d3f1 100644 --- a/llvm/test/CodeGen/AMDGPU/simplify-libcalls.ll +++ b/llvm/test/CodeGen/AMDGPU/simplify-libcalls.ll @@ -278,7 +278,7 @@ entry: ; GCN-LABEL: {{^}}define amdgpu_kernel void @test_pow_half ; GCN-POSTLINK: call fast float @_Z3powff(float %tmp, float 5.000000e-01) -; GCN-PRELINK: %__pow2sqrt = tail call fast float @_Z4sqrtf(float %tmp) +; GCN-PRELINK: %__pow2sqrt = tail call fast float @llvm.sqrt.f32(float %tmp) define amdgpu_kernel void @test_pow_half(ptr addrspace(1) nocapture %a) { entry: %arrayidx = getelementptr inbounds float, ptr addrspace(1) %a, i64 1 @@ -476,7 +476,7 @@ declare float @_Z5rootnfi(float, i32) ; GCN-LABEL: {{^}}define amdgpu_kernel void @test_rootn_2 ; GCN-POSTLINK: call fast float @_Z5rootnfi(float %tmp, i32 2) -; GCN-PRELINK: %__rootn2sqrt = tail call fast float @_Z4sqrtf(float %tmp) +; GCN-PRELINK: %__rootn2sqrt = tail call fast float @llvm.sqrt.f32(float %tmp) define amdgpu_kernel void @test_rootn_2(ptr addrspace(1) nocapture %a) { entry: %tmp = load float, ptr addrspace(1) %a, align 4 @@ -838,5 +838,5 @@ entry: ; GCN-PRELINK: declare float @_Z4cbrtf(float) local_unnamed_addr #[[$NOUNWIND_READONLY:[0-9]+]] ; GCN-PRELINK-DAG: attributes #[[$NOUNWIND]] = { nounwind } -; GCN-PRELINK-DAG: attributes #[[$NOUNWIND_READONLY]] = { nofree nounwind memory(read) } +; GCN-PRELINK-DAG: attributes #[[$NOUNWIND_READONLY]] = { nounwind memory(read) } attributes #0 = { nounwind } -- GitLab From e11b17a4ed90e74147594012207fc35a60515944 Mon Sep 17 00:00:00 2001 From: Vlad Serebrennikov Date: Wed, 17 Apr 2024 09:51:24 +0300 Subject: [PATCH 015/862] [clang][NFC] Refactor `Sema::CheckedConversionKind` Convert it to scoped enum, and move it to namespace scope to enable forward declarations. --- clang/include/clang/Sema/Sema.h | 65 +++++++++++++++---------- clang/lib/Sema/SemaCast.cpp | 73 ++++++++++++++--------------- clang/lib/Sema/SemaExpr.cpp | 10 ++-- clang/lib/Sema/SemaExprCXX.cpp | 8 ++-- clang/lib/Sema/SemaExprObjC.cpp | 45 +++++++++--------- clang/lib/Sema/SemaInit.cpp | 10 ++-- clang/lib/Sema/SemaOverload.cpp | 10 ++-- clang/lib/Sema/SemaPseudoObject.cpp | 2 +- 8 files changed, 119 insertions(+), 104 deletions(-) diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h index 091c5c02f75d..281e3b91de1d 100644 --- a/clang/include/clang/Sema/Sema.h +++ b/clang/include/clang/Sema/Sema.h @@ -437,6 +437,20 @@ enum class CXXSpecialMemberKind { Invalid }; +/// The kind of conversion being performed. +enum class CheckedConversionKind { + /// An implicit conversion. + Implicit, + /// A C-style cast. + CStyleCast, + /// A functional-style cast. + FunctionalCast, + /// A cast other than a C-style cast. + OtherCast, + /// A conversion for an operand of a builtin overloaded operator. + ForBuiltinOverloadedOp +}; + /// Sema - This implements semantic analysis and AST building for C. /// \nosubgrouping class Sema final : public SemaBase { @@ -700,28 +714,27 @@ public: void checkTypeSupport(QualType Ty, SourceLocation Loc, ValueDecl *D = nullptr); - /// The kind of conversion being performed. - enum CheckedConversionKind { - /// An implicit conversion. - CCK_ImplicitConversion, - /// A C-style cast. - CCK_CStyleCast, - /// A functional-style cast. - CCK_FunctionalCast, - /// A cast other than a C-style cast. - CCK_OtherCast, - /// A conversion for an operand of a builtin overloaded operator. - CCK_ForBuiltinOverloadedOp - }; + // /// The kind of conversion being performed. + // enum CheckedConversionKind { + // /// An implicit conversion. + // CCK_ImplicitConversion, + // /// A C-style cast. + // CCK_CStyleCast, + // /// A functional-style cast. + // CCK_FunctionalCast, + // /// A cast other than a C-style cast. + // CCK_OtherCast, + // /// A conversion for an operand of a builtin overloaded operator. + // CCK_ForBuiltinOverloadedOp + // }; /// ImpCastExprToType - If Expr is not of type 'Type', insert an implicit /// cast. If there is already an implicit cast, merge into the existing one. /// If isLvalue, the result of the cast is an lvalue. - ExprResult - ImpCastExprToType(Expr *E, QualType Type, CastKind CK, - ExprValueKind VK = VK_PRValue, - const CXXCastPath *BasePath = nullptr, - CheckedConversionKind CCK = CCK_ImplicitConversion); + ExprResult ImpCastExprToType( + Expr *E, QualType Type, CastKind CK, ExprValueKind VK = VK_PRValue, + const CXXCastPath *BasePath = nullptr, + CheckedConversionKind CCK = CheckedConversionKind::Implicit); /// ScalarTypeToBooleanCastKind - Returns the cast kind corresponding /// to the conversion from scalar type ScalarTy to the Boolean type. @@ -1781,8 +1794,9 @@ public: public: static bool isCast(CheckedConversionKind CCK) { - return CCK == CCK_CStyleCast || CCK == CCK_FunctionalCast || - CCK == CCK_OtherCast; + return CCK == CheckedConversionKind::CStyleCast || + CCK == CheckedConversionKind::FunctionalCast || + CCK == CheckedConversionKind::OtherCast; } /// ActOnCXXNamedCast - Parse @@ -6739,11 +6753,10 @@ public: bool IsStringLiteralToNonConstPointerConversion(Expr *From, QualType ToType); - ExprResult - PerformImplicitConversion(Expr *From, QualType ToType, - const ImplicitConversionSequence &ICS, - AssignmentAction Action, - CheckedConversionKind CCK = CCK_ImplicitConversion); + ExprResult PerformImplicitConversion( + Expr *From, QualType ToType, const ImplicitConversionSequence &ICS, + AssignmentAction Action, + CheckedConversionKind CCK = CheckedConversionKind::Implicit); ExprResult PerformImplicitConversion(Expr *From, QualType ToType, const StandardConversionSequence &SCS, AssignmentAction Action, @@ -7064,7 +7077,7 @@ public: ExprResult PerformQualificationConversion( Expr *E, QualType Ty, ExprValueKind VK = VK_PRValue, - CheckedConversionKind CCK = CCK_ImplicitConversion); + CheckedConversionKind CCK = CheckedConversionKind::Implicit); bool CanPerformCopyInitialization(const InitializedEntity &Entity, ExprResult Init); diff --git a/clang/lib/Sema/SemaCast.cpp b/clang/lib/Sema/SemaCast.cpp index b0c28531fe87..126fd3797417 100644 --- a/clang/lib/Sema/SemaCast.cpp +++ b/clang/lib/Sema/SemaCast.cpp @@ -155,7 +155,7 @@ namespace { Self.CheckCastAlign(SrcExpr.get(), DestType, OpRange); } - void checkObjCConversion(Sema::CheckedConversionKind CCK) { + void checkObjCConversion(CheckedConversionKind CCK) { assert(Self.getLangOpts().allowsNonTrivialObjCLifetimeQualifiers()); Expr *src = SrcExpr.get(); @@ -248,18 +248,14 @@ static TryCastResult TryStaticMemberPointerUpcast(Sema &Self, ExprResult &SrcExp CastKind &Kind, CXXCastPath &BasePath); -static TryCastResult TryStaticImplicitCast(Sema &Self, ExprResult &SrcExpr, - QualType DestType, - Sema::CheckedConversionKind CCK, - SourceRange OpRange, - unsigned &msg, CastKind &Kind, - bool ListInitialization); +static TryCastResult +TryStaticImplicitCast(Sema &Self, ExprResult &SrcExpr, QualType DestType, + CheckedConversionKind CCK, SourceRange OpRange, + unsigned &msg, CastKind &Kind, bool ListInitialization); static TryCastResult TryStaticCast(Sema &Self, ExprResult &SrcExpr, - QualType DestType, - Sema::CheckedConversionKind CCK, - SourceRange OpRange, - unsigned &msg, CastKind &Kind, - CXXCastPath &BasePath, + QualType DestType, CheckedConversionKind CCK, + SourceRange OpRange, unsigned &msg, + CastKind &Kind, CXXCastPath &BasePath, bool ListInitialization); static TryCastResult TryConstCast(Sema &Self, ExprResult &SrcExpr, QualType DestType, bool CStyle, @@ -1223,7 +1219,7 @@ void CastOperation::CheckReinterpretCast() { if (isValidCast(tcr)) { if (Self.getLangOpts().allowsNonTrivialObjCLifetimeQualifiers()) - checkObjCConversion(Sema::CCK_OtherCast); + checkObjCConversion(CheckedConversionKind::OtherCast); DiagnoseReinterpretUpDownCast(Self, SrcExpr.get(), DestType, OpRange); if (unsigned DiagID = checkCastFunctionType(Self, SrcExpr, DestType)) @@ -1274,9 +1270,9 @@ void CastOperation::CheckStaticCast() { } unsigned msg = diag::err_bad_cxx_cast_generic; - TryCastResult tcr - = TryStaticCast(Self, SrcExpr, DestType, Sema::CCK_OtherCast, OpRange, msg, - Kind, BasePath, /*ListInitialization=*/false); + TryCastResult tcr = + TryStaticCast(Self, SrcExpr, DestType, CheckedConversionKind::OtherCast, + OpRange, msg, Kind, BasePath, /*ListInitialization=*/false); if (tcr != TC_Success && msg != 0) { if (SrcExpr.isInvalid()) return; @@ -1296,7 +1292,7 @@ void CastOperation::CheckStaticCast() { if (Kind == CK_BitCast) checkCastAlign(); if (Self.getLangOpts().allowsNonTrivialObjCLifetimeQualifiers()) - checkObjCConversion(Sema::CCK_OtherCast); + checkObjCConversion(CheckedConversionKind::OtherCast); } else { SrcExpr = ExprError(); } @@ -1317,14 +1313,13 @@ static bool IsAddressSpaceConversion(QualType SrcType, QualType DestType) { /// possible. If @p CStyle, ignore access restrictions on hierarchy casting /// and casting away constness. static TryCastResult TryStaticCast(Sema &Self, ExprResult &SrcExpr, - QualType DestType, - Sema::CheckedConversionKind CCK, + QualType DestType, CheckedConversionKind CCK, SourceRange OpRange, unsigned &msg, CastKind &Kind, CXXCastPath &BasePath, bool ListInitialization) { // Determine whether we have the semantics of a C-style cast. - bool CStyle - = (CCK == Sema::CCK_CStyleCast || CCK == Sema::CCK_FunctionalCast); + bool CStyle = (CCK == CheckedConversionKind::CStyleCast || + CCK == CheckedConversionKind::FunctionalCast); // The order the tests is not entirely arbitrary. There is one conversion // that can be handled in two different ways. Given: @@ -1884,11 +1879,11 @@ TryStaticMemberPointerUpcast(Sema &Self, ExprResult &SrcExpr, QualType SrcType, /// /// An expression e can be explicitly converted to a type T using a /// @c static_cast if the declaration "T t(e);" is well-formed [...]. -TryCastResult -TryStaticImplicitCast(Sema &Self, ExprResult &SrcExpr, QualType DestType, - Sema::CheckedConversionKind CCK, - SourceRange OpRange, unsigned &msg, - CastKind &Kind, bool ListInitialization) { +TryCastResult TryStaticImplicitCast(Sema &Self, ExprResult &SrcExpr, + QualType DestType, + CheckedConversionKind CCK, + SourceRange OpRange, unsigned &msg, + CastKind &Kind, bool ListInitialization) { if (DestType->isRecordType()) { if (Self.RequireCompleteType(OpRange.getBegin(), DestType, diag::err_bad_cast_incomplete) || @@ -1900,13 +1895,14 @@ TryStaticImplicitCast(Sema &Self, ExprResult &SrcExpr, QualType DestType, } InitializedEntity Entity = InitializedEntity::InitializeTemporary(DestType); - InitializationKind InitKind - = (CCK == Sema::CCK_CStyleCast) - ? InitializationKind::CreateCStyleCast(OpRange.getBegin(), OpRange, - ListInitialization) - : (CCK == Sema::CCK_FunctionalCast) - ? InitializationKind::CreateFunctionalCast(OpRange, ListInitialization) - : InitializationKind::CreateCast(OpRange); + InitializationKind InitKind = + (CCK == CheckedConversionKind::CStyleCast) + ? InitializationKind::CreateCStyleCast(OpRange.getBegin(), OpRange, + ListInitialization) + : (CCK == CheckedConversionKind::FunctionalCast) + ? InitializationKind::CreateFunctionalCast(OpRange, + ListInitialization) + : InitializationKind::CreateCast(OpRange); Expr *SrcExprRaw = SrcExpr.get(); // FIXME: Per DR242, we should check for an implicit conversion sequence // or for a constructor that could be invoked by direct-initialization @@ -1918,8 +1914,8 @@ TryStaticImplicitCast(Sema &Self, ExprResult &SrcExpr, QualType DestType, // There is no other way that works. // On the other hand, if we're checking a C-style cast, we've still got // the reinterpret_cast way. - bool CStyle - = (CCK == Sema::CCK_CStyleCast || CCK == Sema::CCK_FunctionalCast); + bool CStyle = (CCK == CheckedConversionKind::CStyleCast || + CCK == CheckedConversionKind::FunctionalCast); if (InitSeq.Failed() && (CStyle || !DestType->isReferenceType())) return TC_NotApplicable; @@ -2814,8 +2810,9 @@ void CastOperation::CheckCXXCStyleCast(bool FunctionalStyle, if (isValidCast(tcr)) Kind = CK_NoOp; - Sema::CheckedConversionKind CCK = - FunctionalStyle ? Sema::CCK_FunctionalCast : Sema::CCK_CStyleCast; + CheckedConversionKind CCK = FunctionalStyle + ? CheckedConversionKind::FunctionalCast + : CheckedConversionKind::CStyleCast; if (tcr == TC_NotApplicable) { tcr = TryAddressSpaceCast(Self, SrcExpr, DestType, /*CStyle*/ true, msg, Kind); @@ -3201,7 +3198,7 @@ void CastOperation::CheckCStyleCast() { // ARC imposes extra restrictions on casts. if (Self.getLangOpts().allowsNonTrivialObjCLifetimeQualifiers()) { - checkObjCConversion(Sema::CCK_CStyleCast); + checkObjCConversion(CheckedConversionKind::CStyleCast); if (SrcExpr.isInvalid()) return; diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp index 7c3faba0f788..d2c77ad61644 100644 --- a/clang/lib/Sema/SemaExpr.cpp +++ b/clang/lib/Sema/SemaExpr.cpp @@ -10177,8 +10177,9 @@ Sema::CheckSingleAssignmentConstraints(QualType LHSType, ExprResult &CallerRHS, // diagnostics and just checking for errors, e.g., during overload // resolution, return Incompatible to indicate the failure. if (getLangOpts().allowsNonTrivialObjCLifetimeQualifiers() && - CheckObjCConversion(SourceRange(), Ty, E, CCK_ImplicitConversion, - Diagnose, DiagnoseCFAudited) != ACR_okay) { + CheckObjCConversion(SourceRange(), Ty, E, + CheckedConversionKind::Implicit, Diagnose, + DiagnoseCFAudited) != ACR_okay) { if (!Diagnose) return Incompatible; } @@ -12899,14 +12900,15 @@ QualType Sema::CheckCompareOperands(ExprResult &LHS, ExprResult &RHS, Expr *E = LHS.get(); if (getLangOpts().ObjCAutoRefCount) CheckObjCConversion(SourceRange(), RHSType, E, - CCK_ImplicitConversion); + CheckedConversionKind::Implicit); LHS = ImpCastExprToType(E, RHSType, RPT ? CK_BitCast :CK_CPointerToObjCPointerCast); } else { Expr *E = RHS.get(); if (getLangOpts().ObjCAutoRefCount) - CheckObjCConversion(SourceRange(), LHSType, E, CCK_ImplicitConversion, + CheckObjCConversion(SourceRange(), LHSType, E, + CheckedConversionKind::Implicit, /*Diagnose=*/true, /*DiagnoseCFAudited=*/false, Opc); RHS = ImpCastExprToType(E, LHSType, diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp index 74ed3fe7bd52..f4a91ececfbb 100644 --- a/clang/lib/Sema/SemaExprCXX.cpp +++ b/clang/lib/Sema/SemaExprCXX.cpp @@ -4250,7 +4250,8 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType, AssignmentAction Action, CheckedConversionKind CCK) { // C++ [over.match.oper]p7: [...] operands of class type are converted [...] - if (CCK == CCK_ForBuiltinOverloadedOp && !From->getType()->isRecordType()) + if (CCK == CheckedConversionKind::ForBuiltinOverloadedOp && + !From->getType()->isRecordType()) return From; switch (ICS.getKind()) { @@ -4311,7 +4312,7 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType, // C++ [over.match.oper]p7: // [...] the second standard conversion sequence of a user-defined // conversion sequence is not applied. - if (CCK == CCK_ForBuiltinOverloadedOp) + if (CCK == CheckedConversionKind::ForBuiltinOverloadedOp) return From; return PerformImplicitConversion(From, ToType, ICS.UserDefined.After, @@ -4352,7 +4353,8 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType, const StandardConversionSequence& SCS, AssignmentAction Action, CheckedConversionKind CCK) { - bool CStyle = (CCK == CCK_CStyleCast || CCK == CCK_FunctionalCast); + bool CStyle = (CCK == CheckedConversionKind::CStyleCast || + CCK == CheckedConversionKind::FunctionalCast); // Overall FIXME: we are recomputing too many types here and doing far too // much extra work. What this means is that we need to keep track of more diff --git a/clang/lib/Sema/SemaExprObjC.cpp b/clang/lib/Sema/SemaExprObjC.cpp index 3148f0db6e20..b13a9d426983 100644 --- a/clang/lib/Sema/SemaExprObjC.cpp +++ b/clang/lib/Sema/SemaExprObjC.cpp @@ -3745,22 +3745,22 @@ bool Sema::isKnownName(StringRef name) { template static void addFixitForObjCARCConversion( - Sema &S, DiagBuilderT &DiagB, Sema::CheckedConversionKind CCK, + Sema &S, DiagBuilderT &DiagB, CheckedConversionKind CCK, SourceLocation afterLParen, QualType castType, Expr *castExpr, Expr *realCast, const char *bridgeKeyword, const char *CFBridgeName) { // We handle C-style and implicit casts here. switch (CCK) { - case Sema::CCK_ImplicitConversion: - case Sema::CCK_ForBuiltinOverloadedOp: - case Sema::CCK_CStyleCast: - case Sema::CCK_OtherCast: + case CheckedConversionKind::Implicit: + case CheckedConversionKind::ForBuiltinOverloadedOp: + case CheckedConversionKind::CStyleCast: + case CheckedConversionKind::OtherCast: break; - case Sema::CCK_FunctionalCast: + case CheckedConversionKind::FunctionalCast: return; } if (CFBridgeName) { - if (CCK == Sema::CCK_OtherCast) { + if (CCK == CheckedConversionKind::OtherCast) { if (const CXXNamedCastExpr *NCE = dyn_cast(realCast)) { SourceRange range(NCE->getOperatorLoc(), NCE->getAngleBrackets().getEnd()); @@ -3805,9 +3805,9 @@ static void addFixitForObjCARCConversion( return; } - if (CCK == Sema::CCK_CStyleCast) { + if (CCK == CheckedConversionKind::CStyleCast) { DiagB.AddFixItHint(FixItHint::CreateInsertion(afterLParen, bridgeKeyword)); - } else if (CCK == Sema::CCK_OtherCast) { + } else if (CCK == CheckedConversionKind::OtherCast) { if (const CXXNamedCastExpr *NCE = dyn_cast(realCast)) { std::string castCode = "("; castCode += bridgeKeyword; @@ -3866,12 +3866,12 @@ static ObjCBridgeRelatedAttr *ObjCBridgeRelatedAttrFromType(QualType T, return nullptr; } -static void -diagnoseObjCARCConversion(Sema &S, SourceRange castRange, - QualType castType, ARCConversionTypeClass castACTC, - Expr *castExpr, Expr *realCast, - ARCConversionTypeClass exprACTC, - Sema::CheckedConversionKind CCK) { +static void diagnoseObjCARCConversion(Sema &S, SourceRange castRange, + QualType castType, + ARCConversionTypeClass castACTC, + Expr *castExpr, Expr *realCast, + ARCConversionTypeClass exprACTC, + CheckedConversionKind CCK) { SourceLocation loc = (castRange.isValid() ? castRange.getBegin() : castExpr->getExprLoc()); @@ -3927,7 +3927,7 @@ diagnoseObjCARCConversion(Sema &S, SourceRange castRange, assert(CreateRule != ACC_bottom && "This cast should already be accepted."); if (CreateRule != ACC_plusOne) { - auto DiagB = (CCK != Sema::CCK_OtherCast) + auto DiagB = (CCK != CheckedConversionKind::OtherCast) ? S.Diag(noteLoc, diag::note_arc_bridge) : S.Diag(noteLoc, diag::note_arc_cstyle_bridge); @@ -3937,7 +3937,7 @@ diagnoseObjCARCConversion(Sema &S, SourceRange castRange, } if (CreateRule != ACC_plusZero) { - auto DiagB = (CCK == Sema::CCK_OtherCast && !br) + auto DiagB = (CCK == CheckedConversionKind::OtherCast && !br) ? S.Diag(noteLoc, diag::note_arc_cstyle_bridge_transfer) << castExprType : S.Diag(br ? castExpr->getExprLoc() : noteLoc, @@ -3968,7 +3968,7 @@ diagnoseObjCARCConversion(Sema &S, SourceRange castRange, assert(CreateRule != ACC_bottom && "This cast should already be accepted."); if (CreateRule != ACC_plusOne) { - auto DiagB = (CCK != Sema::CCK_OtherCast) + auto DiagB = (CCK != CheckedConversionKind::OtherCast) ? S.Diag(noteLoc, diag::note_arc_bridge) : S.Diag(noteLoc, diag::note_arc_cstyle_bridge); addFixitForObjCARCConversion(S, DiagB, CCK, afterLParen, @@ -3977,7 +3977,7 @@ diagnoseObjCARCConversion(Sema &S, SourceRange castRange, } if (CreateRule != ACC_plusZero) { - auto DiagB = (CCK == Sema::CCK_OtherCast && !br) + auto DiagB = (CCK == CheckedConversionKind::OtherCast && !br) ? S.Diag(noteLoc, diag::note_arc_cstyle_bridge_retained) << castType : S.Diag(br ? castExpr->getExprLoc() : noteLoc, @@ -4403,7 +4403,8 @@ Sema::CheckObjCConversion(SourceRange castRange, QualType castType, // Check for viability and report error if casting an rvalue to a // life-time qualifier. if (castACTC == ACTC_retainable && - (CCK == CCK_CStyleCast || CCK == CCK_OtherCast) && + (CCK == CheckedConversionKind::CStyleCast || + CCK == CheckedConversionKind::OtherCast) && castType != castExprType) { const Type *DT = castType.getTypePtr(); QualType QDT = castType; @@ -4517,11 +4518,11 @@ void Sema::diagnoseARCUnbridgedCast(Expr *e) { if (CStyleCastExpr *cast = dyn_cast(realCast)) { castRange = SourceRange(cast->getLParenLoc(), cast->getRParenLoc()); castType = cast->getTypeAsWritten(); - CCK = CCK_CStyleCast; + CCK = CheckedConversionKind::CStyleCast; } else if (ExplicitCastExpr *cast = dyn_cast(realCast)) { castRange = cast->getTypeInfoAsWritten()->getTypeLoc().getSourceRange(); castType = cast->getTypeAsWritten(); - CCK = CCK_OtherCast; + CCK = CheckedConversionKind::OtherCast; } else { llvm_unreachable("Unexpected ImplicitCastExpr"); } diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp index fb7a80ab0284..e86f7578ff0c 100644 --- a/clang/lib/Sema/SemaInit.cpp +++ b/clang/lib/Sema/SemaInit.cpp @@ -9057,11 +9057,11 @@ ExprResult InitializationSequence::Perform(Sema &S, } } - Sema::CheckedConversionKind CCK - = Kind.isCStyleCast()? Sema::CCK_CStyleCast - : Kind.isFunctionalCast()? Sema::CCK_FunctionalCast - : Kind.isExplicitCast()? Sema::CCK_OtherCast - : Sema::CCK_ImplicitConversion; + CheckedConversionKind CCK = + Kind.isCStyleCast() ? CheckedConversionKind::CStyleCast + : Kind.isFunctionalCast() ? CheckedConversionKind::FunctionalCast + : Kind.isExplicitCast() ? CheckedConversionKind::OtherCast + : CheckedConversionKind::Implicit; ExprResult CurInitExprRes = S.PerformImplicitConversion(CurInit.get(), Step->Type, *Step->ICS, getAssignmentAction(Entity), CCK); diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp index 227ef564ba3e..adc319e97b76 100644 --- a/clang/lib/Sema/SemaOverload.cpp +++ b/clang/lib/Sema/SemaOverload.cpp @@ -14506,7 +14506,7 @@ Sema::CreateOverloadedUnaryOp(SourceLocation OpLoc, UnaryOperatorKind Opc, // operator node. ExprResult InputRes = PerformImplicitConversion( Input, Best->BuiltinParamTypes[0], Best->Conversions[0], AA_Passing, - CCK_ForBuiltinOverloadedOp); + CheckedConversionKind::ForBuiltinOverloadedOp); if (InputRes.isInvalid()) return ExprError(); Input = InputRes.get(); @@ -14989,14 +14989,14 @@ ExprResult Sema::CreateOverloadedBinOp(SourceLocation OpLoc, // operator node. ExprResult ArgsRes0 = PerformImplicitConversion( Args[0], Best->BuiltinParamTypes[0], Best->Conversions[0], - AA_Passing, CCK_ForBuiltinOverloadedOp); + AA_Passing, CheckedConversionKind::ForBuiltinOverloadedOp); if (ArgsRes0.isInvalid()) return ExprError(); Args[0] = ArgsRes0.get(); ExprResult ArgsRes1 = PerformImplicitConversion( Args[1], Best->BuiltinParamTypes[1], Best->Conversions[1], - AA_Passing, CCK_ForBuiltinOverloadedOp); + AA_Passing, CheckedConversionKind::ForBuiltinOverloadedOp); if (ArgsRes1.isInvalid()) return ExprError(); Args[1] = ArgsRes1.get(); @@ -15367,14 +15367,14 @@ ExprResult Sema::CreateOverloadedArraySubscriptExpr(SourceLocation LLoc, // operator node. ExprResult ArgsRes0 = PerformImplicitConversion( Args[0], Best->BuiltinParamTypes[0], Best->Conversions[0], - AA_Passing, CCK_ForBuiltinOverloadedOp); + AA_Passing, CheckedConversionKind::ForBuiltinOverloadedOp); if (ArgsRes0.isInvalid()) return ExprError(); Args[0] = ArgsRes0.get(); ExprResult ArgsRes1 = PerformImplicitConversion( Args[1], Best->BuiltinParamTypes[1], Best->Conversions[1], - AA_Passing, CCK_ForBuiltinOverloadedOp); + AA_Passing, CheckedConversionKind::ForBuiltinOverloadedOp); if (ArgsRes1.isInvalid()) return ExprError(); Args[1] = ArgsRes1.get(); diff --git a/clang/lib/Sema/SemaPseudoObject.cpp b/clang/lib/Sema/SemaPseudoObject.cpp index 82774760b34d..c6a0a182d358 100644 --- a/clang/lib/Sema/SemaPseudoObject.cpp +++ b/clang/lib/Sema/SemaPseudoObject.cpp @@ -1136,7 +1136,7 @@ static void CheckKeyForObjCARCConversion(Sema &S, QualType ContainerT, return; QualType T = Getter->parameters()[0]->getType(); S.CheckObjCConversion(Key->getSourceRange(), T, Key, - Sema::CCK_ImplicitConversion); + CheckedConversionKind::Implicit); } bool ObjCSubscriptOpBuilder::findAtIndexGetter() { -- GitLab From 49b209d0d1833a339e66735e1288c1805224603e Mon Sep 17 00:00:00 2001 From: Jan Patrick Lehr Date: Wed, 17 Apr 2024 09:11:43 +0200 Subject: [PATCH 016/862] Revert "[Libomptarget] Rework Record & Replay to be a plugin member" (#89028) Reverts llvm/llvm-project#88928 This broke the AMDGPU buildbots: https://lab.llvm.org/buildbot/#/builders/193/builds/50201 https://lab.llvm.org/staging/#/builders/185/builds/5565 https://lab.llvm.org/buildbot/#/builders/259/builds/2955 --- .../common/include/PluginInterface.h | 11 ------ .../common/src/PluginInterface.cpp | 39 +++++++------------ 2 files changed, 15 insertions(+), 35 deletions(-) diff --git a/openmp/libomptarget/plugins-nextgen/common/include/PluginInterface.h b/openmp/libomptarget/plugins-nextgen/common/include/PluginInterface.h index 7f05464f36c1..79e8464bfda5 100644 --- a/openmp/libomptarget/plugins-nextgen/common/include/PluginInterface.h +++ b/openmp/libomptarget/plugins-nextgen/common/include/PluginInterface.h @@ -45,8 +45,6 @@ #include "llvm/Support/raw_ostream.h" #include "llvm/TargetParser/Triple.h" -struct RecordReplayTy; - namespace llvm { namespace omp { namespace target { @@ -1033,12 +1031,6 @@ struct GenericPluginTy { return *RPCServer; } - /// Get a reference to the R&R interface for this plugin. - RecordReplayTy &getRecordAndReplay() const { - assert(RecordReplay && "R&R not initialized"); - return *RecordReplay; - } - /// Get the OpenMP requires flags set for this plugin. int64_t getRequiresFlags() const { return RequiresFlags; } @@ -1228,9 +1220,6 @@ private: /// The interface between the plugin and the GPU for host services. RPCServerTy *RPCServer; - - /// The interface into the record-and-replay functionality. - RecordReplayTy *RecordReplay; }; namespace Plugin { diff --git a/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp b/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp index 6df9798f12e3..b5f3c45c835f 100644 --- a/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp +++ b/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp @@ -362,6 +362,8 @@ public: } }; +static RecordReplayTy RecordReplay; + // Extract the mapping of host function pointers to device function pointers // from the entry table. Functions marked as 'indirect' in OpenMP will have // offloading entries generated for them which map the host's function pointer @@ -471,8 +473,7 @@ GenericKernelTy::getKernelLaunchEnvironment( // Ctor/Dtor have no arguments, replaying uses the original kernel launch // environment. Older versions of the compiler do not generate a kernel // launch environment. - if (isCtorOrDtor() || - GenericDevice.Plugin.getRecordAndReplay().isReplaying() || + if (isCtorOrDtor() || RecordReplay.isReplaying() || Version < OMP_KERNEL_ARG_MIN_VERSION_WITH_DYN_PTR) return nullptr; @@ -561,7 +562,6 @@ Error GenericKernelTy::launch(GenericDeviceTy &GenericDevice, void **ArgPtrs, // Record the kernel description after we modified the argument count and num // blocks/threads. - RecordReplayTy &RecordReplay = GenericDevice.Plugin.getRecordAndReplay(); if (RecordReplay.isRecording()) { RecordReplay.saveImage(getName(), getImage()); RecordReplay.saveKernelInput(getName(), getImage()); @@ -839,6 +839,9 @@ Error GenericDeviceTy::deinit(GenericPluginTy &Plugin) { delete MemoryManager; MemoryManager = nullptr; + if (RecordReplay.isRecordingOrReplaying()) + RecordReplay.deinit(); + if (RPCServer) if (auto Err = RPCServer->deinitDevice(*this)) return Err; @@ -855,7 +858,6 @@ Error GenericDeviceTy::deinit(GenericPluginTy &Plugin) { return deinitImpl(); } - Expected GenericDeviceTy::loadBinary(GenericPluginTy &Plugin, const __tgt_device_image *InputTgtImage) { @@ -890,8 +892,7 @@ GenericDeviceTy::loadBinary(GenericPluginTy &Plugin, return std::move(Err); // Setup the global device memory pool if needed. - if (!Plugin.getRecordAndReplay().isReplaying() && - shouldSetupDeviceMemoryPool()) { + if (!RecordReplay.isReplaying() && shouldSetupDeviceMemoryPool()) { uint64_t HeapSize; auto SizeOrErr = getDeviceHeapSize(HeapSize); if (SizeOrErr) { @@ -1306,8 +1307,8 @@ Expected GenericDeviceTy::dataAlloc(int64_t Size, void *HostPtr, TargetAllocTy Kind) { void *Alloc = nullptr; - if (Plugin.getRecordAndReplay().isRecordingOrReplaying()) - return Plugin.getRecordAndReplay().alloc(Size); + if (RecordReplay.isRecordingOrReplaying()) + return RecordReplay.alloc(Size); switch (Kind) { case TARGET_ALLOC_DEFAULT: @@ -1343,7 +1344,7 @@ Expected GenericDeviceTy::dataAlloc(int64_t Size, void *HostPtr, Error GenericDeviceTy::dataDelete(void *TgtPtr, TargetAllocTy Kind) { // Free is a noop when recording or replaying. - if (Plugin.getRecordAndReplay().isRecordingOrReplaying()) + if (RecordReplay.isRecordingOrReplaying()) return Plugin::success(); int Res; @@ -1395,7 +1396,6 @@ Error GenericDeviceTy::launchKernel(void *EntryPtr, void **ArgPtrs, ptrdiff_t *ArgOffsets, KernelArgsTy &KernelArgs, __tgt_async_info *AsyncInfo) { - RecordReplayTy &RecordReplay = Plugin.getRecordAndReplay(); AsyncInfoWrapperTy AsyncInfoWrapper( *this, RecordReplay.isRecordingOrReplaying() ? nullptr : AsyncInfo); @@ -1495,9 +1495,6 @@ Error GenericPluginTy::init() { RPCServer = new RPCServerTy(*this); assert(RPCServer && "Invalid RPC server"); - RecordReplay = new RecordReplayTy(); - assert(RecordReplay && "Invalid Record and Replay handler"); - return Plugin::success(); } @@ -1511,9 +1508,6 @@ Error GenericPluginTy::deinit() { assert(!Devices[DeviceId] && "Device was not deinitialized"); } - if (RecordReplay && RecordReplay->isRecordingOrReplaying()) - RecordReplay->deinit(); - // There is no global handler if no device is available. if (GlobalHandler) delete GlobalHandler; @@ -1521,9 +1515,6 @@ Error GenericPluginTy::deinit() { if (RPCServer) delete RPCServer; - if (RecordReplay) - delete RecordReplay; - // Perform last deinitializations on the plugin. return deinitImpl(); } @@ -1639,12 +1630,12 @@ int32_t GenericPluginTy::initialize_record_replay(int32_t DeviceId, isRecord ? RecordReplayTy::RRStatusTy::RRRecording : RecordReplayTy::RRStatusTy::RRReplaying; - if (auto Err = RecordReplay->init(&Device, MemorySize, VAddr, Status, - SaveOutput, ReqPtrArgOffset)) { + if (auto Err = RecordReplay.init(&Device, MemorySize, VAddr, Status, + SaveOutput, ReqPtrArgOffset)) { REPORT("WARNING RR did not intialize RR-properly with %lu bytes" "(Error: %s)\n", MemorySize, toString(std::move(Err)).data()); - RecordReplay->setStatus(RecordReplayTy::RRStatusTy::RRDeactivated); + RecordReplay.setStatus(RecordReplayTy::RRStatusTy::RRDeactivated); if (!isRecord) { return OFFLOAD_FAIL; @@ -1993,8 +1984,8 @@ int32_t GenericPluginTy::get_global(__tgt_device_binary Binary, uint64_t Size, assert(DevicePtr && "Invalid device global's address"); // Save the loaded globals if we are recording. - if (getRecordAndReplay().isRecording()) - getRecordAndReplay().addEntry(Name, Size, *DevicePtr); + if (RecordReplay.isRecording()) + RecordReplay.addEntry(Name, Size, *DevicePtr); return OFFLOAD_SUCCESS; } -- GitLab From 9f3334e9932fc9b55cd3590b140913222454c031 Mon Sep 17 00:00:00 2001 From: Matthias Springer Date: Wed, 17 Apr 2024 09:20:55 +0200 Subject: [PATCH 017/862] [mlir][SparseTensor] Add missing dependent dialect to pass (#88870) This commit fixes the following error when stopping the sparse compiler pipeline after bufferization (e.g., with `test-analysis-only`): ``` LLVM ERROR: Building op `vector.print` but it isn't known in this MLIRContext: the dialect may not be loaded or this operation hasn't been added by the dialect. See also https://mlir.llvm.org/getting_started/Faq/#registered-loaded-dependent-whats-up-with-dialects-management ``` --- mlir/include/mlir/Dialect/SparseTensor/Transforms/Passes.td | 1 + .../Transforms/SparsificationAndBufferizationPass.cpp | 1 + .../Integration/Dialect/SparseTensor/CPU/sparse_block3d.mlir | 4 ++++ 3 files changed, 6 insertions(+) diff --git a/mlir/include/mlir/Dialect/SparseTensor/Transforms/Passes.td b/mlir/include/mlir/Dialect/SparseTensor/Transforms/Passes.td index 4706d5ba2f21..2f844cee5ff5 100644 --- a/mlir/include/mlir/Dialect/SparseTensor/Transforms/Passes.td +++ b/mlir/include/mlir/Dialect/SparseTensor/Transforms/Passes.td @@ -460,6 +460,7 @@ def SparsificationAndBufferization : Pass<"sparsification-and-bufferization", "M "memref::MemRefDialect", "scf::SCFDialect", "sparse_tensor::SparseTensorDialect", + "vector::VectorDialect" ]; } diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparsificationAndBufferizationPass.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparsificationAndBufferizationPass.cpp index f497be6e48eb..3a8972072ac3 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/SparsificationAndBufferizationPass.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparsificationAndBufferizationPass.cpp @@ -24,6 +24,7 @@ #include "mlir/Dialect/SCF/IR/SCF.h" #include "mlir/Dialect/SparseTensor/IR/SparseTensor.h" #include "mlir/Dialect/SparseTensor/Transforms/Passes.h" +#include "mlir/Dialect/Vector/IR/VectorOps.h" #include "mlir/Pass/PassManager.h" #include "mlir/Transforms/Passes.h" diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_block3d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_block3d.mlir index 2ff73923c832..467b671500e1 100755 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_block3d.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_block3d.mlir @@ -30,6 +30,10 @@ // Do the same run, but now with direct IR generation and VLA vectorization. // RUN: %if mlir_arm_sve_tests %{ %{compile_sve} | %{run_sve} | FileCheck %s %} +// Test that test-bufferization-analysis-only works. This option is useful +// for understanding why buffer copies were inserted. +// RUN: mlir-opt %s --sparsifier="test-bufferization-analysis-only" -o /dev/null + #Sparse1 = #sparse_tensor.encoding<{ map = (i, j, k) -> ( j : compressed, -- GitLab From 889dfd4ab35892840f2bd2d6d7fed6fac025e18e Mon Sep 17 00:00:00 2001 From: Guillaume Chatelet Date: Wed, 17 Apr 2024 10:04:22 +0200 Subject: [PATCH 018/862] [libc][msan] Fix "non-constexpr function '__msan_unpoison' cannot be used in a constant expression" (#88719) Prior to this patch, calling `cpp::bit_cast` in `constexpr` expressions under `-fsanitize=memory` would fail with the following message "non-constexpr function '__msan_unpoison' cannot be used in a constant expression". This patch makes sure that the `__msan_unpoison` expression is guarded by `!__builtin_is_constant_evaluated()`. --- libc/src/__support/macros/sanitizer.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/libc/src/__support/macros/sanitizer.h b/libc/src/__support/macros/sanitizer.h index bd9b62b7121a..baf44f7996ca 100644 --- a/libc/src/__support/macros/sanitizer.h +++ b/libc/src/__support/macros/sanitizer.h @@ -47,14 +47,13 @@ // Functions to unpoison memory //----------------------------------------------------------------------------- -#if defined(LIBC_HAVE_MEMORY_SANITIZER) && __has_builtin(__builtin_constant_p) +#if defined(LIBC_HAVE_MEMORY_SANITIZER) // Only perform MSAN unpoison in non-constexpr context. #include #define MSAN_UNPOISON(addr, size) \ do { \ - if (!__builtin_constant_p(*addr)) { \ + if (!__builtin_is_constant_evaluated()) \ __msan_unpoison(addr, size); \ - } \ } while (0) #else #define MSAN_UNPOISON(ptr, size) -- GitLab From 17b86d5978af8d171fa28763a9e5eba3ce93713a Mon Sep 17 00:00:00 2001 From: Phoebe Wang Date: Wed, 17 Apr 2024 15:59:37 +0800 Subject: [PATCH 019/862] [X86][NFC] Add test cases for pr88958 --- llvm/test/CodeGen/X86/combine-ptest.ll | 42 ++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/llvm/test/CodeGen/X86/combine-ptest.ll b/llvm/test/CodeGen/X86/combine-ptest.ll index 337edef96bee..3a695bfc6234 100644 --- a/llvm/test/CodeGen/X86/combine-ptest.ll +++ b/llvm/test/CodeGen/X86/combine-ptest.ll @@ -397,6 +397,48 @@ define i1 @PR38788(<4 x i32> %0, <4 x i32> %1) { ret i1 %7 } +define i32 @PR88958_1(ptr %0, <2 x i64> %1) { +; SSE-LABEL: PR88958_1: +; SSE: # %bb.0: +; SSE-NEXT: movdqa (%rdi), %xmm1 +; SSE-NEXT: xorl %eax, %eax +; SSE-NEXT: ptest %xmm0, %xmm1 +; SSE-NEXT: sete %al +; SSE-NEXT: retq +; +; AVX-LABEL: PR88958_1: +; AVX: # %bb.0: +; AVX-NEXT: vmovdqa (%rdi), %xmm1 +; AVX-NEXT: xorl %eax, %eax +; AVX-NEXT: vptest %xmm0, %xmm1 +; AVX-NEXT: sete %al +; AVX-NEXT: retq + %3 = load <2 x i64>, ptr %0 + %4 = tail call i32 @llvm.x86.sse41.ptestz(<2 x i64> %3, <2 x i64> %1) + ret i32 %4 +} + +define i32 @PR88958_2(ptr %0, <2 x i64> %1) { +; SSE-LABEL: PR88958_2: +; SSE: # %bb.0: +; SSE-NEXT: movdqa (%rdi), %xmm1 +; SSE-NEXT: xorl %eax, %eax +; SSE-NEXT: ptest %xmm0, %xmm1 +; SSE-NEXT: setb %al +; SSE-NEXT: retq +; +; AVX-LABEL: PR88958_2: +; AVX: # %bb.0: +; AVX-NEXT: vmovdqa (%rdi), %xmm1 +; AVX-NEXT: xorl %eax, %eax +; AVX-NEXT: vptest %xmm0, %xmm1 +; AVX-NEXT: setb %al +; AVX-NEXT: retq + %3 = load <2 x i64>, ptr %0 + %4 = tail call i32 @llvm.x86.sse41.ptestc(<2 x i64> %3, <2 x i64> %1) + ret i32 %4 +} + declare i32 @llvm.x86.sse41.ptestz(<2 x i64>, <2 x i64>) nounwind readnone declare i32 @llvm.x86.sse41.ptestc(<2 x i64>, <2 x i64>) nounwind readnone declare i32 @llvm.x86.sse41.ptestnzc(<2 x i64>, <2 x i64>) nounwind readnone -- GitLab From d1a69e4a6ee0b04778da7728123c47eef2290564 Mon Sep 17 00:00:00 2001 From: shamithoke <152091883+shamithoke@users.noreply.github.com> Date: Wed, 17 Apr 2024 14:16:10 +0530 Subject: [PATCH 020/862] Move gfni for bitreverse check out of SSSE3. (#88938) For lowering bitreverse using GFNI, the check is put under SSSE3. This can be pulled out of SSSE3. Co-authored-by: shami --- llvm/lib/Target/X86/X86ISelLowering.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index f16a751a166d..27107f554fcc 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -1276,6 +1276,13 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, setOperationAction(ISD::STRICT_FDIV, MVT::v2f64, Legal); } + if (Subtarget.hasGFNI()) { + setOperationAction(ISD::BITREVERSE, MVT::i8, Custom); + setOperationAction(ISD::BITREVERSE, MVT::i16, Custom); + setOperationAction(ISD::BITREVERSE, MVT::i32, Custom); + setOperationAction(ISD::BITREVERSE, MVT::i64, Custom); + } + if (!Subtarget.useSoftFloat() && Subtarget.hasSSSE3()) { setOperationAction(ISD::ABS, MVT::v16i8, Legal); setOperationAction(ISD::ABS, MVT::v8i16, Legal); @@ -1286,13 +1293,6 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, setOperationAction(ISD::CTLZ, VT, Custom); } - if (Subtarget.hasGFNI()) { - setOperationAction(ISD::BITREVERSE, MVT::i8, Custom); - setOperationAction(ISD::BITREVERSE, MVT::i16, Custom); - setOperationAction(ISD::BITREVERSE, MVT::i32, Custom); - setOperationAction(ISD::BITREVERSE, MVT::i64, Custom); - } - // These might be better off as horizontal vector ops. setOperationAction(ISD::ADD, MVT::i16, Custom); setOperationAction(ISD::ADD, MVT::i32, Custom); -- GitLab From a16bb0701409376dee3a587ae351a6019d6de4e0 Mon Sep 17 00:00:00 2001 From: David Spickett Date: Wed, 17 Apr 2024 09:16:46 +0000 Subject: [PATCH 021/862] [lldb][test] Improve invalid compiler error message I was debugging space separation issues when passing user arguments and noticed this error is really hard to read in that scenario. Put "" around the invalid compiler name so you can tell whether you have spaces around it that's causing the problem. --- lldb/packages/Python/lldbsuite/test/dotest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lldb/packages/Python/lldbsuite/test/dotest.py b/lldb/packages/Python/lldbsuite/test/dotest.py index 8c29145ecc52..2ec4a840b916 100644 --- a/lldb/packages/Python/lldbsuite/test/dotest.py +++ b/lldb/packages/Python/lldbsuite/test/dotest.py @@ -248,7 +248,7 @@ def parseOptionsAndInitTestdirs(): configuration.compiler = which(args.compiler) if not is_exe(configuration.compiler): logging.error( - "%s is not a valid compiler executable; aborting...", args.compiler + '"%s" is not a valid compiler executable; aborting...', args.compiler ) sys.exit(-1) else: -- GitLab From d9a5aa8e2d755643cf4e7fa86aa831ed226fe54d Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Wed, 17 Apr 2024 18:22:05 +0900 Subject: [PATCH 022/862] [PatternMatch] Do not accept undef elements in m_AllOnes() and friends (#88217) Change all the cstval_pred_ty based PatternMatch helpers (things like m_AllOnes and m_Zero) to only allow poison elements inside vector splats, not undef elements. Historically, we used to represent non-demanded elements in vectors using undef. Nowadays, we use poison instead. As such, I believe that support for undef in vector splats is no longer useful. At the same time, while poison splat elements are pretty much always safe to ignore, this is not generally the case for undef elements. We have existing miscompiles in our tests due to this (see the masked-merge-*.ll tests changed here) and it's easy to miss such cases in the future, now that we write tests using poison instead of undef elements. I think overall, keeping support for undef elements no longer makes sense, and we should drop it. Once this is done consistently, I think we may also consider allowing poison in m_APInt by default, as doing that change is much less risky than doing the same with undef. This change involves a substantial amount of test changes. For most tests, I've just replaced undef with poison, as I don't think there is value in retaining both. For some tests (where the distinction between undef and poison is important), I've duplicated tests. --- llvm/include/llvm/IR/PatternMatch.h | 35 +---- llvm/lib/Analysis/InstructionSimplify.cpp | 23 ++- llvm/lib/IR/Constants.cpp | 2 +- .../InstCombine/InstCombineAndOrXor.cpp | 12 +- .../InstCombine/X86/x86-vector-shifts.ll | 30 ++-- llvm/test/Transforms/InstCombine/abs-1.ll | 16 +- .../Transforms/InstCombine/add-mask-neg.ll | 6 +- llvm/test/Transforms/InstCombine/add.ll | 28 ++-- .../Transforms/InstCombine/and-or-icmps.ll | 61 +++++--- .../test/Transforms/InstCombine/and-xor-or.ll | 4 +- llvm/test/Transforms/InstCombine/and.ll | 86 +++++------ llvm/test/Transforms/InstCombine/and2.ll | 19 ++- llvm/test/Transforms/InstCombine/ashr-lshr.ll | 28 ++-- .../Transforms/InstCombine/ashr-or-mul-abs.ll | 8 +- .../InstCombine/binop-and-shifts.ll | 68 ++++---- .../InstCombine/binop-of-displaced-shifts.ll | 22 +-- ...ern-between-zero-and-positive-threshold.ll | 20 +-- ...nt-low-bit-mask-and-icmp-eq-to-icmp-ule.ll | 6 +- ...nt-low-bit-mask-and-icmp-ne-to-icmp-ugt.ll | 12 +- ...t-low-bit-mask-and-icmp-sge-to-icmp-sle.ll | 10 +- ...t-low-bit-mask-and-icmp-sgt-to-icmp-sgt.ll | 16 +- ...t-low-bit-mask-and-icmp-sle-to-icmp-sle.ll | 10 +- ...t-low-bit-mask-and-icmp-slt-to-icmp-sgt.ll | 16 +- ...t-low-bit-mask-and-icmp-uge-to-icmp-ule.ll | 6 +- ...t-low-bit-mask-and-icmp-ugt-to-icmp-ugt.ll | 12 +- ...t-low-bit-mask-and-icmp-ule-to-icmp-ule.ll | 6 +- ...t-low-bit-mask-and-icmp-ult-to-icmp-ugt.ll | 12 +- ...ze-low-bit-mask-and-icmp-eq-to-icmp-ule.ll | 8 +- ...ze-low-bit-mask-and-icmp-ne-to-icmp-ugt.ll | 8 +- ...low-bit-mask-v2-and-icmp-eq-to-icmp-ule.ll | 20 +-- ...low-bit-mask-v2-and-icmp-ne-to-icmp-ugt.ll | 20 +-- ...low-bit-mask-v4-and-icmp-eq-to-icmp-ule.ll | 8 +- ...low-bit-mask-v4-and-icmp-ne-to-icmp-ugt.ll | 8 +- .../InstCombine/cast-int-icmp-eq-0.ll | 22 +-- .../InstCombine/cast-unsigned-icmp-eqcmp-0.ll | 36 ++--- llvm/test/Transforms/InstCombine/cast.ll | 56 +++---- .../test/Transforms/InstCombine/ctpop-cttz.ll | 6 +- llvm/test/Transforms/InstCombine/ctpop.ll | 17 +- .../Transforms/InstCombine/fabs-as-int.ll | 6 +- llvm/test/Transforms/InstCombine/fabs.ll | 4 +- llvm/test/Transforms/InstCombine/fast-math.ll | 6 +- .../Transforms/InstCombine/fcmp-special.ll | 20 +-- llvm/test/Transforms/InstCombine/fcmp.ll | 26 ++-- llvm/test/Transforms/InstCombine/fdiv.ll | 22 +-- llvm/test/Transforms/InstCombine/fma.ll | 8 +- llvm/test/Transforms/InstCombine/fmul.ll | 42 ++--- .../Transforms/InstCombine/fneg-as-int.ll | 6 +- .../InstCombine/fneg-fabs-as-int.ll | 6 +- llvm/test/Transforms/InstCombine/fneg.ll | 36 ++--- ...c-of-add-of-not-x-and-y-to-sub-x-from-y.ll | 20 +-- .../fold-sub-of-not-to-inc-of-add.ll | 6 +- llvm/test/Transforms/InstCombine/fpcast.ll | 6 +- llvm/test/Transforms/InstCombine/fsub.ll | 14 +- llvm/test/Transforms/InstCombine/funnel.ll | 54 +++---- .../get-lowbitmask-upto-and-including-bit.ll | 20 +-- .../hoist-negation-out-of-bias-calculation.ll | 6 +- .../hoist-not-from-ashr-operand.ll | 8 +- ...al-to-icmp-eq-of-lshr-val-by-bits-and-0.ll | 4 +- ...al-to-icmp-eq-of-lshr-val-by-bits-and-0.ll | 20 +-- ...al-to-icmp-eq-of-lshr-val-by-bits-and-0.ll | 6 +- ...al-to-icmp-ne-of-lshr-val-by-bits-and-0.ll | 6 +- ...al-to-icmp-ne-of-lshr-val-by-bits-and-0.ll | 4 +- ...al-to-icmp-ne-of-lshr-val-by-bits-and-0.ll | 20 +-- llvm/test/Transforms/InstCombine/icmp.ll | 62 ++++---- .../integer-round-up-pow2-alignment.ll | 70 ++++----- ...rt-variable-mask-in-masked-merge-vector.ll | 30 ++-- .../InstCombine/lshr-and-negC-icmpeq-zero.ll | 20 +-- .../lshr-and-signbit-icmpeq-zero.ll | 20 +-- .../InstCombine/masked-merge-add.ll | 17 +- .../Transforms/InstCombine/masked-merge-or.ll | 17 +- .../InstCombine/masked-merge-xor.ll | 17 +- .../Transforms/InstCombine/min-positive.ll | 8 +- .../Transforms/InstCombine/minmax-fold.ll | 11 +- .../InstCombine/minmax-intrinsics.ll | 40 ++--- .../InstCombine/mul-inseltpoison.ll | 22 +-- llvm/test/Transforms/InstCombine/mul.ll | 30 ++-- llvm/test/Transforms/InstCombine/not-add.ll | 16 +- llvm/test/Transforms/InstCombine/not.ll | 4 +- ...of-two-or-zero-when-comparing-with-zero.ll | 34 ++-- .../InstCombine/operand-complexity.ll | 22 +-- llvm/test/Transforms/InstCombine/or.ll | 52 +++---- ...nput-masking-after-truncation-variant-b.ll | 20 +-- ...nput-masking-after-truncation-variant-c.ll | 20 +-- ...nput-masking-after-truncation-variant-d.ll | 20 +-- ...dant-left-shift-input-masking-variant-b.ll | 20 +-- ...dant-left-shift-input-masking-variant-c.ll | 12 +- ...dant-left-shift-input-masking-variant-d.ll | 12 +- llvm/test/Transforms/InstCombine/pr53357.ll | 8 +- ...nput-masking-after-truncation-variant-b.ll | 16 +- ...nput-masking-after-truncation-variant-c.ll | 20 +-- ...nput-masking-after-truncation-variant-d.ll | 20 +-- ...dant-left-shift-input-masking-variant-b.ll | 18 +-- ...dant-left-shift-input-masking-variant-c.ll | 12 +- ...dant-left-shift-input-masking-variant-d.ll | 10 +- .../reuse-constant-from-select-in-icmp.ll | 26 ++-- llvm/test/Transforms/InstCombine/rotate.ll | 57 +++---- .../InstCombine/saturating-add-sub.ll | 18 +-- .../InstCombine/select-of-bittest.ll | 89 ++++++----- llvm/test/Transforms/InstCombine/select.ll | 33 ++-- .../Transforms/InstCombine/select_meta.ll | 8 +- .../set-lowbits-mask-canonicalize.ll | 20 +-- llvm/test/Transforms/InstCombine/sext.ll | 28 ++-- .../shift-amount-reassociation-in-bittest.ll | 64 ++++---- ...ount-reassociation-with-truncation-ashr.ll | 26 ++-- ...ount-reassociation-with-truncation-lshr.ll | 32 ++-- .../InstCombine/shift-amount-reassociation.ll | 26 ++-- .../Transforms/InstCombine/shift-logic.ll | 88 +++++------ .../InstCombine/shl-and-negC-icmpeq-zero.ll | 20 +-- .../shl-and-signbit-icmpeq-zero.ll | 20 +-- .../signmask-of-sext-vs-of-shl-of-zext.ll | 42 +++-- llvm/test/Transforms/InstCombine/sub-not.ll | 8 +- llvm/test/Transforms/InstCombine/sub.ll | 44 +++--- .../InstCombine/trunc-inseltpoison.ll | 98 ++++++------ .../InstCombine/trunc-shift-trunc.ll | 16 +- llvm/test/Transforms/InstCombine/trunc.ll | 102 ++++++------ ...k-of-overflow-check-via-udiv-of-allones.ll | 6 +- ...-mul-overflow-check-via-udiv-of-allones.ll | 6 +- ...signext-of-variable-high-bit-extraction.ll | 12 +- llvm/test/Transforms/InstCombine/vec_sext.ll | 14 +- .../InstCombine/vector-casts-inseltpoison.ll | 20 +-- .../Transforms/InstCombine/vector-casts.ll | 20 +-- .../Transforms/InstCombine/vector-urem.ll | 18 +-- .../test/Transforms/InstCombine/vector-xor.ll | 66 ++++---- .../InstCombine/zext-bool-add-sub.ll | 28 ++-- llvm/test/Transforms/InstSimplify/AndOrXor.ll | 36 ++--- llvm/test/Transforms/InstSimplify/call.ll | 10 +- llvm/test/Transforms/InstSimplify/compare.ll | 66 ++++---- ...constantfold-add-nuw-allones-to-allones.ll | 8 +- .../constantfold-shl-nuw-C-to-C.ll | 8 +- llvm/test/Transforms/InstSimplify/div.ll | 20 +-- .../InstSimplify/fast-math-strictfp.ll | 68 ++++---- .../test/Transforms/InstSimplify/fast-math.ll | 60 ++++---- llvm/test/Transforms/InstSimplify/fdiv.ll | 6 +- .../floating-point-arithmetic-strictfp.ll | 54 +++---- .../InstSimplify/floating-point-arithmetic.ll | 38 ++--- .../InstSimplify/floating-point-compare.ll | 30 ++-- .../Transforms/InstSimplify/fminmax-folds.ll | 36 ++--- llvm/test/Transforms/InstSimplify/fp-nan.ll | 13 +- .../InstSimplify/icmp-bool-constant.ll | 18 +-- .../InstSimplify/icmp-not-bool-constant.ll | 36 ++--- llvm/test/Transforms/InstSimplify/ldexp.ll | 5 +- llvm/test/Transforms/InstSimplify/mul.ll | 6 +- llvm/test/Transforms/InstSimplify/negate.ll | 12 +- llvm/test/Transforms/InstSimplify/or.ll | 73 +++++++-- llvm/test/Transforms/InstSimplify/ptrmask.ll | 3 +- llvm/test/Transforms/InstSimplify/rem.ll | 6 +- .../InstSimplify/saturating-add-sub.ll | 8 +- llvm/test/Transforms/InstSimplify/sdiv.ll | 6 +- .../InstSimplify/select-inseltpoison.ll | 38 ++--- llvm/test/Transforms/InstSimplify/select.ll | 38 ++--- llvm/test/Transforms/InstSimplify/shift.ll | 35 +++-- llvm/test/Transforms/InstSimplify/srem.ll | 6 +- llvm/test/Transforms/InstSimplify/sub.ll | 6 +- llvm/test/Transforms/InstSimplify/xor.ll | 20 ++- llvm/test/Transforms/Reassociate/inverses.ll | 6 +- llvm/test/Transforms/Reassociate/negation.ll | 8 +- llvm/unittests/IR/ConstantsTest.cpp | 39 ++++- llvm/unittests/IR/PatternMatch.cpp | 145 ++++++++++++++---- 158 files changed, 2042 insertions(+), 1839 deletions(-) diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h index 92cb79d54afc..98cc0e503769 100644 --- a/llvm/include/llvm/IR/PatternMatch.h +++ b/llvm/include/llvm/IR/PatternMatch.h @@ -345,7 +345,7 @@ template inline constantint_match m_ConstantInt() { /// This helper class is used to match constant scalars, vector splats, /// and fixed width vectors that satisfy a specified predicate. -/// For fixed width vector constants, undefined elements are ignored. +/// For fixed width vector constants, poison elements are ignored. template struct cstval_pred_ty : public Predicate { template bool match(ITy *V) { @@ -364,19 +364,19 @@ struct cstval_pred_ty : public Predicate { // Non-splat vector constant: check each element for a match. unsigned NumElts = FVTy->getNumElements(); assert(NumElts != 0 && "Constant vector with no elements?"); - bool HasNonUndefElements = false; + bool HasNonPoisonElements = false; for (unsigned i = 0; i != NumElts; ++i) { Constant *Elt = C->getAggregateElement(i); if (!Elt) return false; - if (isa(Elt)) + if (isa(Elt)) continue; auto *CV = dyn_cast(Elt); if (!CV || !this->isValue(CV->getValue())) return false; - HasNonUndefElements = true; + HasNonPoisonElements = true; } - return HasNonUndefElements; + return HasNonPoisonElements; } } return false; @@ -2587,31 +2587,6 @@ m_Not(const ValTy &V) { return m_c_Xor(m_AllOnes(), V); } -template struct NotForbidUndef_match { - ValTy Val; - NotForbidUndef_match(const ValTy &V) : Val(V) {} - - template bool match(OpTy *V) { - // We do not use m_c_Xor because that could match an arbitrary APInt that is - // not -1 as C and then fail to match the other operand if it is -1. - // This code should still work even when both operands are constants. - Value *X; - const APInt *C; - if (m_Xor(m_Value(X), m_APIntForbidUndef(C)).match(V) && C->isAllOnes()) - return Val.match(X); - if (m_Xor(m_APIntForbidUndef(C), m_Value(X)).match(V) && C->isAllOnes()) - return Val.match(X); - return false; - } -}; - -/// Matches a bitwise 'not' as 'xor V, -1' or 'xor -1, V'. For vectors, the -/// constant value must be composed of only -1 scalar elements. -template -inline NotForbidUndef_match m_NotForbidUndef(const ValTy &V) { - return NotForbidUndef_match(V); -} - /// Matches an SMin with LHS and RHS in either order. template inline MaxMin_match diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp index 8955de6375de..06ba5ca4c6b3 100644 --- a/llvm/lib/Analysis/InstructionSimplify.cpp +++ b/llvm/lib/Analysis/InstructionSimplify.cpp @@ -1513,7 +1513,7 @@ static Value *simplifyAShrInst(Value *Op0, Value *Op1, bool IsExact, // -1 >>a X --> -1 // (-1 << X) a>> X --> -1 - // Do not return Op0 because it may contain undef elements if it's a vector. + // We could return the original -1 constant to preserve poison elements. if (match(Op0, m_AllOnes()) || match(Op0, m_Shl(m_AllOnes(), m_Specific(Op1)))) return Constant::getAllOnesValue(Op0->getType()); @@ -2281,7 +2281,7 @@ static Value *simplifyOrLogic(Value *X, Value *Y) { // (B ^ ~A) | (A & B) --> B ^ ~A // (~A ^ B) | (B & A) --> ~A ^ B // (B ^ ~A) | (B & A) --> B ^ ~A - if (match(X, m_c_Xor(m_NotForbidUndef(m_Value(A)), m_Value(B))) && + if (match(X, m_c_Xor(m_Not(m_Value(A)), m_Value(B))) && match(Y, m_c_And(m_Specific(A), m_Specific(B)))) return X; @@ -2298,31 +2298,29 @@ static Value *simplifyOrLogic(Value *X, Value *Y) { // (B & ~A) | ~(A | B) --> ~A // (B & ~A) | ~(B | A) --> ~A Value *NotA; - if (match(X, - m_c_And(m_CombineAnd(m_Value(NotA), m_NotForbidUndef(m_Value(A))), - m_Value(B))) && + if (match(X, m_c_And(m_CombineAnd(m_Value(NotA), m_Not(m_Value(A))), + m_Value(B))) && match(Y, m_Not(m_c_Or(m_Specific(A), m_Specific(B))))) return NotA; // The same is true of Logical And // TODO: This could share the logic of the version above if there was a // version of LogicalAnd that allowed more than just i1 types. - if (match(X, m_c_LogicalAnd( - m_CombineAnd(m_Value(NotA), m_NotForbidUndef(m_Value(A))), - m_Value(B))) && + if (match(X, m_c_LogicalAnd(m_CombineAnd(m_Value(NotA), m_Not(m_Value(A))), + m_Value(B))) && match(Y, m_Not(m_c_LogicalOr(m_Specific(A), m_Specific(B))))) return NotA; // ~(A ^ B) | (A & B) --> ~(A ^ B) // ~(A ^ B) | (B & A) --> ~(A ^ B) Value *NotAB; - if (match(X, m_CombineAnd(m_NotForbidUndef(m_Xor(m_Value(A), m_Value(B))), + if (match(X, m_CombineAnd(m_Not(m_Xor(m_Value(A), m_Value(B))), m_Value(NotAB))) && match(Y, m_c_And(m_Specific(A), m_Specific(B)))) return NotAB; // ~(A & B) | (A ^ B) --> ~(A & B) // ~(A & B) | (B ^ A) --> ~(A & B) - if (match(X, m_CombineAnd(m_NotForbidUndef(m_And(m_Value(A), m_Value(B))), + if (match(X, m_CombineAnd(m_Not(m_And(m_Value(A), m_Value(B))), m_Value(NotAB))) && match(Y, m_c_Xor(m_Specific(A), m_Specific(B)))) return NotAB; @@ -2552,9 +2550,8 @@ static Value *simplifyXorInst(Value *Op0, Value *Op1, const SimplifyQuery &Q, // The 'not' op must contain a complete -1 operand (no undef elements for // vector) for the transform to be safe. Value *NotA; - if (match(X, - m_c_Or(m_CombineAnd(m_NotForbidUndef(m_Value(A)), m_Value(NotA)), - m_Value(B))) && + if (match(X, m_c_Or(m_CombineAnd(m_Not(m_Value(A)), m_Value(NotA)), + m_Value(B))) && match(Y, m_c_And(m_Specific(A), m_Specific(B)))) return NotA; diff --git a/llvm/lib/IR/Constants.cpp b/llvm/lib/IR/Constants.cpp index a5fb497f54ed..45b359a94b3a 100644 --- a/llvm/lib/IR/Constants.cpp +++ b/llvm/lib/IR/Constants.cpp @@ -316,7 +316,7 @@ bool Constant::isElementWiseEqual(Value *Y) const { Constant *C0 = ConstantExpr::getBitCast(const_cast(this), IntTy); Constant *C1 = ConstantExpr::getBitCast(cast(Y), IntTy); Constant *CmpEq = ConstantExpr::getICmp(ICmpInst::ICMP_EQ, C0, C1); - return isa(CmpEq) || match(CmpEq, m_One()); + return isa(CmpEq) || match(CmpEq, m_One()); } static bool diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp index d311690be64f..0f4fbf5bbfbb 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp @@ -2538,6 +2538,8 @@ Instruction *InstCombinerImpl::visitAnd(BinaryOperator &I) { } } + // and(shl(zext(X), Y), SignMask) -> and(sext(X), SignMask) + // where Y is a valid shift amount. if (match(&I, m_And(m_OneUse(m_Shl(m_ZExt(m_Value(X)), m_Value(Y))), m_SignMask())) && match(Y, m_SpecificInt_ICMP( @@ -2546,15 +2548,7 @@ Instruction *InstCombinerImpl::visitAnd(BinaryOperator &I) { Ty->getScalarSizeInBits() - X->getType()->getScalarSizeInBits())))) { auto *SExt = Builder.CreateSExt(X, Ty, X->getName() + ".signext"); - auto *SanitizedSignMask = cast(Op1); - // We must be careful with the undef elements of the sign bit mask, however: - // the mask elt can be undef iff the shift amount for that lane was undef, - // otherwise we need to sanitize undef masks to zero. - SanitizedSignMask = Constant::replaceUndefsWith( - SanitizedSignMask, ConstantInt::getNullValue(Ty->getScalarType())); - SanitizedSignMask = - Constant::mergeUndefsWith(SanitizedSignMask, cast(Y)); - return BinaryOperator::CreateAnd(SExt, SanitizedSignMask); + return BinaryOperator::CreateAnd(SExt, Op1); } if (Instruction *Z = narrowMaskedBinOp(I)) diff --git a/llvm/test/Transforms/InstCombine/X86/x86-vector-shifts.ll b/llvm/test/Transforms/InstCombine/X86/x86-vector-shifts.ll index 4600a6654a36..b1e5fa4f9e1c 100644 --- a/llvm/test/Transforms/InstCombine/X86/x86-vector-shifts.ll +++ b/llvm/test/Transforms/InstCombine/X86/x86-vector-shifts.ll @@ -2032,23 +2032,23 @@ define <4 x i64> @avx2_psrlv_q_256_allbig(<4 x i64> %v) { ret <4 x i64> %1 } -; The shift amount is 0 (the undef lane could be 0), so we return the unshifted input. +; The shift amount is 0 (the poison lane could be 0), so we return the unshifted input. -define <2 x i64> @avx2_psrlv_q_128_undef(<2 x i64> %v) { -; CHECK-LABEL: @avx2_psrlv_q_128_undef( +define <2 x i64> @avx2_psrlv_q_128_poison(<2 x i64> %v) { +; CHECK-LABEL: @avx2_psrlv_q_128_poison( ; CHECK-NEXT: ret <2 x i64> [[V:%.*]] ; - %1 = insertelement <2 x i64> , i64 undef, i64 1 + %1 = insertelement <2 x i64> , i64 poison, i64 1 %2 = tail call <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> %v, <2 x i64> %1) ret <2 x i64> %2 } -define <4 x i64> @avx2_psrlv_q_256_undef(<4 x i64> %v) { -; CHECK-LABEL: @avx2_psrlv_q_256_undef( -; CHECK-NEXT: [[TMP1:%.*]] = lshr <4 x i64> [[V:%.*]], +define <4 x i64> @avx2_psrlv_q_256_poison(<4 x i64> %v) { +; CHECK-LABEL: @avx2_psrlv_q_256_poison( +; CHECK-NEXT: [[TMP1:%.*]] = lshr <4 x i64> [[V:%.*]], ; CHECK-NEXT: ret <4 x i64> [[TMP1]] ; - %1 = insertelement <4 x i64> , i64 undef, i64 0 + %1 = insertelement <4 x i64> , i64 poison, i64 0 %2 = tail call <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64> %v, <4 x i64> %1) ret <4 x i64> %2 } @@ -2435,21 +2435,21 @@ define <4 x i64> @avx2_psllv_q_256_allbig(<4 x i64> %v) { ; The shift amount is 0 (the undef lane could be 0), so we return the unshifted input. -define <2 x i64> @avx2_psllv_q_128_undef(<2 x i64> %v) { -; CHECK-LABEL: @avx2_psllv_q_128_undef( +define <2 x i64> @avx2_psllv_q_128_poison(<2 x i64> %v) { +; CHECK-LABEL: @avx2_psllv_q_128_poison( ; CHECK-NEXT: ret <2 x i64> [[V:%.*]] ; - %1 = insertelement <2 x i64> , i64 undef, i64 1 + %1 = insertelement <2 x i64> , i64 poison, i64 1 %2 = tail call <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64> %v, <2 x i64> %1) ret <2 x i64> %2 } -define <4 x i64> @avx2_psllv_q_256_undef(<4 x i64> %v) { -; CHECK-LABEL: @avx2_psllv_q_256_undef( -; CHECK-NEXT: [[TMP1:%.*]] = shl <4 x i64> [[V:%.*]], +define <4 x i64> @avx2_psllv_q_256_poison(<4 x i64> %v) { +; CHECK-LABEL: @avx2_psllv_q_256_poison( +; CHECK-NEXT: [[TMP1:%.*]] = shl <4 x i64> [[V:%.*]], ; CHECK-NEXT: ret <4 x i64> [[TMP1]] ; - %1 = insertelement <4 x i64> , i64 undef, i64 0 + %1 = insertelement <4 x i64> , i64 poison, i64 0 %2 = tail call <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64> %v, <4 x i64> %1) ret <4 x i64> %2 } diff --git a/llvm/test/Transforms/InstCombine/abs-1.ll b/llvm/test/Transforms/InstCombine/abs-1.ll index 7355c560c820..32bd7a37053e 100644 --- a/llvm/test/Transforms/InstCombine/abs-1.ll +++ b/llvm/test/Transforms/InstCombine/abs-1.ll @@ -63,14 +63,14 @@ define <2 x i8> @abs_canonical_2(<2 x i8> %x) { ret <2 x i8> %abs } -; Even if a constant has undef elements. +; Even if a constant has poison elements. -define <2 x i8> @abs_canonical_2_vec_undef_elts(<2 x i8> %x) { -; CHECK-LABEL: @abs_canonical_2_vec_undef_elts( +define <2 x i8> @abs_canonical_2_vec_poison_elts(<2 x i8> %x) { +; CHECK-LABEL: @abs_canonical_2_vec_poison_elts( ; CHECK-NEXT: [[ABS:%.*]] = call <2 x i8> @llvm.abs.v2i8(<2 x i8> [[X:%.*]], i1 false) ; CHECK-NEXT: ret <2 x i8> [[ABS]] ; - %cmp = icmp sgt <2 x i8> %x, + %cmp = icmp sgt <2 x i8> %x, %neg = sub <2 x i8> zeroinitializer, %x %abs = select <2 x i1> %cmp, <2 x i8> %x, <2 x i8> %neg ret <2 x i8> %abs @@ -208,15 +208,15 @@ define <2 x i8> @nabs_canonical_2(<2 x i8> %x) { ret <2 x i8> %abs } -; Even if a constant has undef elements. +; Even if a constant has poison elements. -define <2 x i8> @nabs_canonical_2_vec_undef_elts(<2 x i8> %x) { -; CHECK-LABEL: @nabs_canonical_2_vec_undef_elts( +define <2 x i8> @nabs_canonical_2_vec_poison_elts(<2 x i8> %x) { +; CHECK-LABEL: @nabs_canonical_2_vec_poison_elts( ; CHECK-NEXT: [[TMP1:%.*]] = call <2 x i8> @llvm.abs.v2i8(<2 x i8> [[X:%.*]], i1 false) ; CHECK-NEXT: [[ABS:%.*]] = sub <2 x i8> zeroinitializer, [[TMP1]] ; CHECK-NEXT: ret <2 x i8> [[ABS]] ; - %cmp = icmp sgt <2 x i8> %x, + %cmp = icmp sgt <2 x i8> %x, %neg = sub <2 x i8> zeroinitializer, %x %abs = select <2 x i1> %cmp, <2 x i8> %neg, <2 x i8> %x ret <2 x i8> %abs diff --git a/llvm/test/Transforms/InstCombine/add-mask-neg.ll b/llvm/test/Transforms/InstCombine/add-mask-neg.ll index 5fad6155d348..0e579f309760 100644 --- a/llvm/test/Transforms/InstCombine/add-mask-neg.ll +++ b/llvm/test/Transforms/InstCombine/add-mask-neg.ll @@ -89,8 +89,8 @@ define <2 x i32> @dec_mask_neg_v2i32(<2 x i32> %X) { ret <2 x i32> %dec } -define <2 x i32> @dec_mask_neg_v2i32_undef(<2 x i32> %X) { -; CHECK-LABEL: @dec_mask_neg_v2i32_undef( +define <2 x i32> @dec_mask_neg_v2i32_poison(<2 x i32> %X) { +; CHECK-LABEL: @dec_mask_neg_v2i32_poison( ; CHECK-NEXT: [[TMP1:%.*]] = add <2 x i32> [[X:%.*]], ; CHECK-NEXT: [[TMP2:%.*]] = xor <2 x i32> [[X]], ; CHECK-NEXT: [[DEC:%.*]] = and <2 x i32> [[TMP1]], [[TMP2]] @@ -98,7 +98,7 @@ define <2 x i32> @dec_mask_neg_v2i32_undef(<2 x i32> %X) { ; %neg = sub <2 x i32> zeroinitializer, %X %mask = and <2 x i32> %neg, %X - %dec = add <2 x i32> %mask, + %dec = add <2 x i32> %mask, ret <2 x i32> %dec } diff --git a/llvm/test/Transforms/InstCombine/add.ll b/llvm/test/Transforms/InstCombine/add.ll index 408b0c6559b0..39b4ad805508 100644 --- a/llvm/test/Transforms/InstCombine/add.ll +++ b/llvm/test/Transforms/InstCombine/add.ll @@ -150,24 +150,24 @@ define i32 @test5_add_nsw(i32 %A, i32 %B) { ret i32 %D } -define <2 x i8> @neg_op0_vec_undef_elt(<2 x i8> %a, <2 x i8> %b) { -; CHECK-LABEL: @neg_op0_vec_undef_elt( +define <2 x i8> @neg_op0_vec_poison_elt(<2 x i8> %a, <2 x i8> %b) { +; CHECK-LABEL: @neg_op0_vec_poison_elt( ; CHECK-NEXT: [[R:%.*]] = sub <2 x i8> [[B:%.*]], [[A:%.*]] ; CHECK-NEXT: ret <2 x i8> [[R]] ; - %nega = sub <2 x i8> , %a + %nega = sub <2 x i8> , %a %r = add <2 x i8> %nega, %b ret <2 x i8> %r } -define <2 x i8> @neg_neg_vec_undef_elt(<2 x i8> %a, <2 x i8> %b) { -; CHECK-LABEL: @neg_neg_vec_undef_elt( +define <2 x i8> @neg_neg_vec_poison_elt(<2 x i8> %a, <2 x i8> %b) { +; CHECK-LABEL: @neg_neg_vec_poison_elt( ; CHECK-NEXT: [[TMP1:%.*]] = add <2 x i8> [[A:%.*]], [[B:%.*]] ; CHECK-NEXT: [[R:%.*]] = sub <2 x i8> zeroinitializer, [[TMP1]] ; CHECK-NEXT: ret <2 x i8> [[R]] ; - %nega = sub <2 x i8> , %a - %negb = sub <2 x i8> , %b + %nega = sub <2 x i8> , %a + %negb = sub <2 x i8> , %b %r = add <2 x i8> %nega, %negb ret <2 x i8> %r } @@ -1196,14 +1196,14 @@ define <2 x i32> @test44_vec_non_matching(<2 x i32> %A) { ret <2 x i32> %C } -define <2 x i32> @test44_vec_undef(<2 x i32> %A) { -; CHECK-LABEL: @test44_vec_undef( -; CHECK-NEXT: [[B:%.*]] = or <2 x i32> [[A:%.*]], -; CHECK-NEXT: [[C:%.*]] = add <2 x i32> [[B]], +define <2 x i32> @test44_vec_poison(<2 x i32> %A) { +; CHECK-LABEL: @test44_vec_poison( +; CHECK-NEXT: [[B:%.*]] = or <2 x i32> [[A:%.*]], +; CHECK-NEXT: [[C:%.*]] = add nsw <2 x i32> [[B]], ; CHECK-NEXT: ret <2 x i32> [[C]] ; - %B = or <2 x i32> %A, - %C = add <2 x i32> %B, + %B = or <2 x i32> %A, + %C = add <2 x i32> %B, ret <2 x i32> %C } @@ -2983,7 +2983,7 @@ define i8 @signum_i8_i8_use3(i8 %x) { ret i8 %r } -; poison/undef is ok to propagate in shift amount +; poison is ok to propagate in shift amount ; complexity canonicalization guarantees that shift is op0 of add define <2 x i5> @signum_v2i5_v2i5(<2 x i5> %x) { diff --git a/llvm/test/Transforms/InstCombine/and-or-icmps.ll b/llvm/test/Transforms/InstCombine/and-or-icmps.ll index 63b11d0c0bc0..c20f48a985b3 100644 --- a/llvm/test/Transforms/InstCombine/and-or-icmps.ll +++ b/llvm/test/Transforms/InstCombine/and-or-icmps.ll @@ -952,8 +952,8 @@ define i1 @substitute_constant_or_ne_uge_commute_logical(i8 %x, i8 %y) { ; Negative test - not safe to substitute vector constant with undef element -define <2 x i1> @substitute_constant_or_ne_slt_swap_vec(<2 x i8> %x, <2 x i8> %y) { -; CHECK-LABEL: @substitute_constant_or_ne_slt_swap_vec( +define <2 x i1> @substitute_constant_or_ne_slt_swap_vec_undef(<2 x i8> %x, <2 x i8> %y) { +; CHECK-LABEL: @substitute_constant_or_ne_slt_swap_vec_undef( ; CHECK-NEXT: [[C1:%.*]] = icmp ne <2 x i8> [[X:%.*]], ; CHECK-NEXT: [[C2:%.*]] = icmp slt <2 x i8> [[Y:%.*]], [[X]] ; CHECK-NEXT: [[R:%.*]] = or <2 x i1> [[C1]], [[C2]] @@ -965,14 +965,29 @@ define <2 x i1> @substitute_constant_or_ne_slt_swap_vec(<2 x i8> %x, <2 x i8> %y ret <2 x i1> %r } +; TODO: The poison case would be valid to fold. + +define <2 x i1> @substitute_constant_or_ne_slt_swap_vec_poison(<2 x i8> %x, <2 x i8> %y) { +; CHECK-LABEL: @substitute_constant_or_ne_slt_swap_vec_poison( +; CHECK-NEXT: [[C1:%.*]] = icmp ne <2 x i8> [[X:%.*]], +; CHECK-NEXT: [[C2:%.*]] = icmp slt <2 x i8> [[Y:%.*]], [[X]] +; CHECK-NEXT: [[R:%.*]] = or <2 x i1> [[C1]], [[C2]] +; CHECK-NEXT: ret <2 x i1> [[R]] +; + %c1 = icmp ne <2 x i8> %x, + %c2 = icmp slt <2 x i8> %y, %x + %r = or <2 x i1> %c1, %c2 + ret <2 x i1> %r +} + define <2 x i1> @substitute_constant_or_ne_slt_swap_vec_logical(<2 x i8> %x, <2 x i8> %y) { ; CHECK-LABEL: @substitute_constant_or_ne_slt_swap_vec_logical( -; CHECK-NEXT: [[C1:%.*]] = icmp ne <2 x i8> [[X:%.*]], +; CHECK-NEXT: [[C1:%.*]] = icmp ne <2 x i8> [[X:%.*]], ; CHECK-NEXT: [[C2:%.*]] = icmp slt <2 x i8> [[Y:%.*]], [[X]] ; CHECK-NEXT: [[R:%.*]] = select <2 x i1> [[C1]], <2 x i1> , <2 x i1> [[C2]] ; CHECK-NEXT: ret <2 x i1> [[R]] ; - %c1 = icmp ne <2 x i8> %x, + %c1 = icmp ne <2 x i8> %x, %c2 = icmp slt <2 x i8> %y, %x %r = select <2 x i1> %c1, <2 x i1> , <2 x i1> %c2 ret <2 x i1> %r @@ -2497,29 +2512,29 @@ define <2 x i1> @icmp_eq_m1_and_eq_m1(<2 x i8> %x, <2 x i8> %y) { ; CHECK-NEXT: [[R:%.*]] = icmp eq <2 x i8> [[TMP1]], ; CHECK-NEXT: ret <2 x i1> [[R]] ; - %rx = icmp eq <2 x i8> %x, - %ry = icmp eq <2 x i8> %y, + %rx = icmp eq <2 x i8> %x, + %ry = icmp eq <2 x i8> %y, %r = and <2 x i1> %rx, %ry ret <2 x i1> %r } -define <2 x i1> @icmp_eq_m1_and_eq_undef_m1(<2 x i8> %x, <2 x i8> %y) { -; CHECK-LABEL: @icmp_eq_m1_and_eq_undef_m1( +define <2 x i1> @icmp_eq_m1_and_eq_poison_m1(<2 x i8> %x, <2 x i8> %y) { +; CHECK-LABEL: @icmp_eq_m1_and_eq_poison_m1( ; CHECK-NEXT: [[TMP1:%.*]] = and <2 x i8> [[X:%.*]], [[Y:%.*]] ; CHECK-NEXT: [[R:%.*]] = icmp eq <2 x i8> [[TMP1]], ; CHECK-NEXT: ret <2 x i1> [[R]] ; - %rx = icmp eq <2 x i8> %x, - %ry = icmp eq <2 x i8> %y, + %rx = icmp eq <2 x i8> %x, + %ry = icmp eq <2 x i8> %y, %r = and <2 x i1> %rx, %ry ret <2 x i1> %r } -define <2 x i1> @icmp_eq_undef_and_eq_m1_m2(<2 x i8> %x, <2 x i8> %y) { -; CHECK-LABEL: @icmp_eq_undef_and_eq_m1_m2( -; CHECK-NEXT: ret <2 x i1> zeroinitializer +define <2 x i1> @icmp_eq_poison_and_eq_m1_m2(<2 x i8> %x, <2 x i8> %y) { +; CHECK-LABEL: @icmp_eq_poison_and_eq_m1_m2( +; CHECK-NEXT: ret <2 x i1> poison ; - %rx = icmp eq <2 x i8> %x, + %rx = icmp eq <2 x i8> %x, %ry = icmp eq <2 x i8> %y, %r = and <2 x i1> %rx, %ry ret <2 x i1> %r @@ -2527,13 +2542,13 @@ define <2 x i1> @icmp_eq_undef_and_eq_m1_m2(<2 x i8> %x, <2 x i8> %y) { define <2 x i1> @icmp_ne_m1_and_ne_m1_fail(<2 x i8> %x, <2 x i8> %y) { ; CHECK-LABEL: @icmp_ne_m1_and_ne_m1_fail( -; CHECK-NEXT: [[RX:%.*]] = icmp ne <2 x i8> [[X:%.*]], -; CHECK-NEXT: [[RY:%.*]] = icmp ne <2 x i8> [[Y:%.*]], +; CHECK-NEXT: [[RX:%.*]] = icmp ne <2 x i8> [[X:%.*]], +; CHECK-NEXT: [[RY:%.*]] = icmp ne <2 x i8> [[Y:%.*]], ; CHECK-NEXT: [[R:%.*]] = and <2 x i1> [[RX]], [[RY]] ; CHECK-NEXT: ret <2 x i1> [[R]] ; - %rx = icmp ne <2 x i8> %x, - %ry = icmp ne <2 x i8> %y, + %rx = icmp ne <2 x i8> %x, + %ry = icmp ne <2 x i8> %y, %r = and <2 x i1> %rx, %ry ret <2 x i1> %r } @@ -2541,13 +2556,13 @@ define <2 x i1> @icmp_ne_m1_and_ne_m1_fail(<2 x i8> %x, <2 x i8> %y) { define <2 x i1> @icmp_eq_m1_or_eq_m1_fail(<2 x i8> %x, <2 x i8> %y) { ; CHECK-LABEL: @icmp_eq_m1_or_eq_m1_fail( -; CHECK-NEXT: [[RX:%.*]] = icmp eq <2 x i8> [[X:%.*]], -; CHECK-NEXT: [[RY:%.*]] = icmp eq <2 x i8> [[Y:%.*]], +; CHECK-NEXT: [[RX:%.*]] = icmp eq <2 x i8> [[X:%.*]], +; CHECK-NEXT: [[RY:%.*]] = icmp eq <2 x i8> [[Y:%.*]], ; CHECK-NEXT: [[R:%.*]] = or <2 x i1> [[RX]], [[RY]] ; CHECK-NEXT: ret <2 x i1> [[R]] ; - %rx = icmp eq <2 x i8> %x, - %ry = icmp eq <2 x i8> %y, + %rx = icmp eq <2 x i8> %x, + %ry = icmp eq <2 x i8> %y, %r = or <2 x i1> %rx, %ry ret <2 x i1> %r } @@ -2560,7 +2575,7 @@ define <2 x i1> @icmp_ne_m1_or_ne_m1(<2 x i8> %x, <2 x i8> %y) { ; CHECK-NEXT: ret <2 x i1> [[R]] ; %rx = icmp ne <2 x i8> %x, - %ry = icmp ne <2 x i8> %y, + %ry = icmp ne <2 x i8> %y, %r = or <2 x i1> %rx, %ry ret <2 x i1> %r } diff --git a/llvm/test/Transforms/InstCombine/and-xor-or.ll b/llvm/test/Transforms/InstCombine/and-xor-or.ll index d072dc15cbb2..b26d6e16c2db 100644 --- a/llvm/test/Transforms/InstCombine/and-xor-or.ll +++ b/llvm/test/Transforms/InstCombine/and-xor-or.ll @@ -843,7 +843,7 @@ define <2 x i6> @not_or_or_not_2i6(<2 x i6> %a0, <2 x i6> %b, <2 x i6> %c) { ; %a = sdiv <2 x i6> , %a0 ; thwart complexity-based canonicalization %not1 = xor <2 x i6> %b, - %not2 = xor <2 x i6> %c, + %not2 = xor <2 x i6> %c, %or1 = or <2 x i6> %a, %not1 %or2 = or <2 x i6> %or1, %not2 ret <2 x i6> %or2 @@ -4018,7 +4018,7 @@ define <2 x i4> @and_orn_xor_commute1(<2 x i4> %a, <2 x i4> %b) { ; CHECK-NEXT: ret <2 x i4> [[R]] ; %xor = xor <2 x i4> %a, %b - %nota = xor <2 x i4> %a, + %nota = xor <2 x i4> %a, %or = or <2 x i4> %nota, %b %r = and <2 x i4> %xor, %or ret <2 x i4> %r diff --git a/llvm/test/Transforms/InstCombine/and.ll b/llvm/test/Transforms/InstCombine/and.ll index ffd8c2a06c86..b5250fc1a784 100644 --- a/llvm/test/Transforms/InstCombine/and.ll +++ b/llvm/test/Transforms/InstCombine/and.ll @@ -752,16 +752,16 @@ define <2 x i64> @test36_uniform(<2 x i32> %X) { ret <2 x i64> %res } -define <2 x i64> @test36_undef(<2 x i32> %X) { -; CHECK-LABEL: @test36_undef( +define <2 x i64> @test36_poison(<2 x i32> %X) { +; CHECK-LABEL: @test36_poison( ; CHECK-NEXT: [[ZEXT:%.*]] = zext <2 x i32> [[X:%.*]] to <2 x i64> -; CHECK-NEXT: [[ZSUB:%.*]] = add <2 x i64> [[ZEXT]], -; CHECK-NEXT: [[RES:%.*]] = and <2 x i64> [[ZSUB]], +; CHECK-NEXT: [[ZSUB:%.*]] = add nuw nsw <2 x i64> [[ZEXT]], +; CHECK-NEXT: [[RES:%.*]] = and <2 x i64> [[ZSUB]], ; CHECK-NEXT: ret <2 x i64> [[RES]] ; %zext = zext <2 x i32> %X to <2 x i64> - %zsub = add <2 x i64> %zext, - %res = and <2 x i64> %zsub, + %zsub = add <2 x i64> %zext, + %res = and <2 x i64> %zsub, ret <2 x i64> %res } @@ -1630,16 +1630,16 @@ define <2 x i8> @lowmask_add_splat(<2 x i8> %x, ptr %p) { ret <2 x i8> %r } -define <2 x i8> @lowmask_add_splat_undef(<2 x i8> %x, ptr %p) { -; CHECK-LABEL: @lowmask_add_splat_undef( -; CHECK-NEXT: [[A:%.*]] = add <2 x i8> [[X:%.*]], +define <2 x i8> @lowmask_add_splat_poison(<2 x i8> %x, ptr %p) { +; CHECK-LABEL: @lowmask_add_splat_poison( +; CHECK-NEXT: [[A:%.*]] = add <2 x i8> [[X:%.*]], ; CHECK-NEXT: store <2 x i8> [[A]], ptr [[P:%.*]], align 2 -; CHECK-NEXT: [[R:%.*]] = and <2 x i8> [[A]], +; CHECK-NEXT: [[R:%.*]] = and <2 x i8> [[X]], ; CHECK-NEXT: ret <2 x i8> [[R]] ; - %a = add <2 x i8> %x, ; 0xc0 + %a = add <2 x i8> %x, ; 0xc0 store <2 x i8> %a, ptr %p - %r = and <2 x i8> %a, ; 0x20 + %r = and <2 x i8> %a, ; 0x20 ret <2 x i8> %r } @@ -1679,14 +1679,14 @@ define <2 x i8> @flip_masked_bit_uniform(<2 x i8> %A) { ret <2 x i8> %C } -define <2 x i8> @flip_masked_bit_undef(<2 x i8> %A) { -; CHECK-LABEL: @flip_masked_bit_undef( +define <2 x i8> @flip_masked_bit_poison(<2 x i8> %A) { +; CHECK-LABEL: @flip_masked_bit_poison( ; CHECK-NEXT: [[TMP1:%.*]] = xor <2 x i8> [[A:%.*]], -; CHECK-NEXT: [[C:%.*]] = and <2 x i8> [[TMP1]], +; CHECK-NEXT: [[C:%.*]] = and <2 x i8> [[TMP1]], ; CHECK-NEXT: ret <2 x i8> [[C]] ; - %B = add <2 x i8> %A, - %C = and <2 x i8> %B, + %B = add <2 x i8> %A, + %C = and <2 x i8> %B, ret <2 x i8> %C } @@ -2004,7 +2004,7 @@ define i16 @invert_signbit_splat_mask_use2(i8 %x, i16 %y) { ret i16 %r } -; extra use of sext is ok +; extra use of sext is ok define i16 @invert_signbit_splat_mask_use3(i8 %x, i16 %y) { ; CHECK-LABEL: @invert_signbit_splat_mask_use3( @@ -2120,41 +2120,40 @@ define <3 x i16> @shl_lshr_pow2_const_case1_non_uniform_vec_negative(<3 x i16> % ret <3 x i16> %r } -define <3 x i16> @shl_lshr_pow2_const_case1_undef1_vec(<3 x i16> %x) { -; CHECK-LABEL: @shl_lshr_pow2_const_case1_undef1_vec( +define <3 x i16> @shl_lshr_pow2_const_case1_poison1_vec(<3 x i16> %x) { +; CHECK-LABEL: @shl_lshr_pow2_const_case1_poison1_vec( ; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <3 x i16> [[X:%.*]], ; CHECK-NEXT: [[R:%.*]] = select <3 x i1> [[TMP1]], <3 x i16> , <3 x i16> zeroinitializer ; CHECK-NEXT: ret <3 x i16> [[R]] ; - %shl = shl <3 x i16> , %x + %shl = shl <3 x i16> , %x %lshr = lshr <3 x i16> %shl, %r = and <3 x i16> %lshr, ret <3 x i16> %r } -define <3 x i16> @shl_lshr_pow2_const_case1_undef2_vec(<3 x i16> %x) { -; CHECK-LABEL: @shl_lshr_pow2_const_case1_undef2_vec( -; CHECK-NEXT: [[SHL:%.*]] = shl <3 x i16> , [[X:%.*]] -; CHECK-NEXT: [[LSHR:%.*]] = lshr <3 x i16> [[SHL]], -; CHECK-NEXT: [[R:%.*]] = and <3 x i16> [[LSHR]], +define <3 x i16> @shl_lshr_pow2_const_case1_poison2_vec(<3 x i16> %x) { +; CHECK-LABEL: @shl_lshr_pow2_const_case1_poison2_vec( +; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <3 x i16> [[X:%.*]], +; CHECK-NEXT: [[R:%.*]] = select <3 x i1> [[TMP1]], <3 x i16> , <3 x i16> zeroinitializer ; CHECK-NEXT: ret <3 x i16> [[R]] ; %shl = shl <3 x i16> , %x - %lshr = lshr <3 x i16> %shl, + %lshr = lshr <3 x i16> %shl, %r = and <3 x i16> %lshr, ret <3 x i16> %r } -define <3 x i16> @shl_lshr_pow2_const_case1_undef3_vec(<3 x i16> %x) { -; CHECK-LABEL: @shl_lshr_pow2_const_case1_undef3_vec( +define <3 x i16> @shl_lshr_pow2_const_case1_poison3_vec(<3 x i16> %x) { +; CHECK-LABEL: @shl_lshr_pow2_const_case1_poison3_vec( ; CHECK-NEXT: [[SHL:%.*]] = shl <3 x i16> , [[X:%.*]] ; CHECK-NEXT: [[LSHR:%.*]] = lshr <3 x i16> [[SHL]], -; CHECK-NEXT: [[R:%.*]] = and <3 x i16> [[LSHR]], +; CHECK-NEXT: [[R:%.*]] = and <3 x i16> [[LSHR]], ; CHECK-NEXT: ret <3 x i16> [[R]] ; %shl = shl <3 x i16> , %x %lshr = lshr <3 x i16> %shl, - %r = and <3 x i16> %lshr, + %r = and <3 x i16> %lshr, ret <3 x i16> %r } @@ -2417,40 +2416,41 @@ define <3 x i16> @lshr_shl_pow2_const_case1_non_uniform_vec_negative(<3 x i16> % ret <3 x i16> %r } -define <3 x i16> @lshr_shl_pow2_const_case1_undef1_vec(<3 x i16> %x) { -; CHECK-LABEL: @lshr_shl_pow2_const_case1_undef1_vec( +define <3 x i16> @lshr_shl_pow2_const_case1_poison1_vec(<3 x i16> %x) { +; CHECK-LABEL: @lshr_shl_pow2_const_case1_poison1_vec( ; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <3 x i16> [[X:%.*]], ; CHECK-NEXT: [[R:%.*]] = select <3 x i1> [[TMP1]], <3 x i16> , <3 x i16> zeroinitializer ; CHECK-NEXT: ret <3 x i16> [[R]] ; - %lshr = lshr <3 x i16> , %x + %lshr = lshr <3 x i16> , %x %shl = shl <3 x i16> %lshr, %r = and <3 x i16> %shl, ret <3 x i16> %r } -define <3 x i16> @lshr_shl_pow2_const_case1_undef2_vec(<3 x i16> %x) { -; CHECK-LABEL: @lshr_shl_pow2_const_case1_undef2_vec( -; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <3 x i16> [[X:%.*]], -; CHECK-NEXT: [[R:%.*]] = select <3 x i1> [[TMP1]], <3 x i16> , <3 x i16> zeroinitializer +define <3 x i16> @lshr_shl_pow2_const_case1_poison2_vec(<3 x i16> %x) { +; CHECK-LABEL: @lshr_shl_pow2_const_case1_poison2_vec( +; CHECK-NEXT: [[LSHR:%.*]] = lshr <3 x i16> , [[X:%.*]] +; CHECK-NEXT: [[SHL:%.*]] = shl <3 x i16> [[LSHR]], +; CHECK-NEXT: [[R:%.*]] = and <3 x i16> [[SHL]], ; CHECK-NEXT: ret <3 x i16> [[R]] ; %lshr = lshr <3 x i16> , %x - %shl = shl <3 x i16> %lshr, + %shl = shl <3 x i16> %lshr, %r = and <3 x i16> %shl, ret <3 x i16> %r } -define <3 x i16> @lshr_shl_pow2_const_case1_undef3_vec(<3 x i16> %x) { -; CHECK-LABEL: @lshr_shl_pow2_const_case1_undef3_vec( +define <3 x i16> @lshr_shl_pow2_const_case1_poison3_vec(<3 x i16> %x) { +; CHECK-LABEL: @lshr_shl_pow2_const_case1_poison3_vec( ; CHECK-NEXT: [[LSHR:%.*]] = lshr <3 x i16> , [[X:%.*]] ; CHECK-NEXT: [[SHL:%.*]] = shl <3 x i16> [[LSHR]], -; CHECK-NEXT: [[R:%.*]] = and <3 x i16> [[SHL]], +; CHECK-NEXT: [[R:%.*]] = and <3 x i16> [[SHL]], ; CHECK-NEXT: ret <3 x i16> [[R]] ; %lshr = lshr <3 x i16> , %x %shl = shl <3 x i16> %lshr, - %r = and <3 x i16> %shl, + %r = and <3 x i16> %shl, ret <3 x i16> %r } diff --git a/llvm/test/Transforms/InstCombine/and2.ll b/llvm/test/Transforms/InstCombine/and2.ll index 73bdadc86710..104486e7638f 100644 --- a/llvm/test/Transforms/InstCombine/and2.ll +++ b/llvm/test/Transforms/InstCombine/and2.ll @@ -168,14 +168,14 @@ define <2 x i8> @and1_shl1_is_cmp_eq_0_vec(<2 x i8> %x) { ret <2 x i8> %and } -define <2 x i8> @and1_shl1_is_cmp_eq_0_vec_undef(<2 x i8> %x) { -; CHECK-LABEL: @and1_shl1_is_cmp_eq_0_vec_undef( +define <2 x i8> @and1_shl1_is_cmp_eq_0_vec_poison(<2 x i8> %x) { +; CHECK-LABEL: @and1_shl1_is_cmp_eq_0_vec_poison( ; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <2 x i8> [[X:%.*]], zeroinitializer ; CHECK-NEXT: [[AND:%.*]] = zext <2 x i1> [[TMP1]] to <2 x i8> ; CHECK-NEXT: ret <2 x i8> [[AND]] ; - %sh = shl <2 x i8> , %x - %and = and <2 x i8> %sh, + %sh = shl <2 x i8> , %x + %and = and <2 x i8> %sh, ret <2 x i8> %and } @@ -215,14 +215,13 @@ define <2 x i8> @and1_lshr1_is_cmp_eq_0_vec(<2 x i8> %x) { ret <2 x i8> %and } -define <2 x i8> @and1_lshr1_is_cmp_eq_0_vec_undef(<2 x i8> %x) { -; CHECK-LABEL: @and1_lshr1_is_cmp_eq_0_vec_undef( -; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <2 x i8> [[X:%.*]], zeroinitializer -; CHECK-NEXT: [[AND:%.*]] = zext <2 x i1> [[TMP1]] to <2 x i8> +define <2 x i8> @and1_lshr1_is_cmp_eq_0_vec_poison(<2 x i8> %x) { +; CHECK-LABEL: @and1_lshr1_is_cmp_eq_0_vec_poison( +; CHECK-NEXT: [[AND:%.*]] = lshr <2 x i8> , [[X:%.*]] ; CHECK-NEXT: ret <2 x i8> [[AND]] ; - %sh = lshr <2 x i8> , %x - %and = and <2 x i8> %sh, + %sh = lshr <2 x i8> , %x + %and = and <2 x i8> %sh, ret <2 x i8> %and } diff --git a/llvm/test/Transforms/InstCombine/ashr-lshr.ll b/llvm/test/Transforms/InstCombine/ashr-lshr.ll index 60fa5b2597ba..ac206dc7999d 100644 --- a/llvm/test/Transforms/InstCombine/ashr-lshr.ll +++ b/llvm/test/Transforms/InstCombine/ashr-lshr.ll @@ -229,24 +229,24 @@ define <2 x i32> @ashr_lshr_inv_nonsplat_vec(<2 x i32> %x, <2 x i32> %y) { ret <2 x i32> %ret } -define <2 x i32> @ashr_lshr_vec_undef(<2 x i32> %x, <2 x i32> %y) { -; CHECK-LABEL: @ashr_lshr_vec_undef( +define <2 x i32> @ashr_lshr_vec_poison(<2 x i32> %x, <2 x i32> %y) { +; CHECK-LABEL: @ashr_lshr_vec_poison( ; CHECK-NEXT: [[CMP12:%.*]] = ashr <2 x i32> [[X:%.*]], [[Y:%.*]] ; CHECK-NEXT: ret <2 x i32> [[CMP12]] ; - %cmp = icmp sgt <2 x i32> %x, + %cmp = icmp sgt <2 x i32> %x, %l = lshr <2 x i32> %x, %y %r = ashr exact <2 x i32> %x, %y %ret = select <2 x i1> %cmp, <2 x i32> %l, <2 x i32> %r ret <2 x i32> %ret } -define <2 x i32> @ashr_lshr_vec_undef2(<2 x i32> %x, <2 x i32> %y) { -; CHECK-LABEL: @ashr_lshr_vec_undef2( +define <2 x i32> @ashr_lshr_vec_poison2(<2 x i32> %x, <2 x i32> %y) { +; CHECK-LABEL: @ashr_lshr_vec_poison2( ; CHECK-NEXT: [[CMP1:%.*]] = ashr exact <2 x i32> [[X:%.*]], [[Y:%.*]] ; CHECK-NEXT: ret <2 x i32> [[CMP1]] ; - %cmp = icmp slt <2 x i32> %x, + %cmp = icmp slt <2 x i32> %x, %l = lshr exact <2 x i32> %x, %y %r = ashr exact <2 x i32> %x, %y %ret = select <2 x i1> %cmp, <2 x i32> %r, <2 x i32> %l @@ -498,14 +498,14 @@ define <3 x i42> @lshr_sub_nsw_splat(<3 x i42> %x, <3 x i42> %y) { ret <3 x i42> %shr } -define <3 x i42> @lshr_sub_nsw_splat_undef(<3 x i42> %x, <3 x i42> %y) { -; CHECK-LABEL: @lshr_sub_nsw_splat_undef( +define <3 x i42> @lshr_sub_nsw_splat_poison(<3 x i42> %x, <3 x i42> %y) { +; CHECK-LABEL: @lshr_sub_nsw_splat_poison( ; CHECK-NEXT: [[SUB:%.*]] = sub nsw <3 x i42> [[X:%.*]], [[Y:%.*]] -; CHECK-NEXT: [[SHR:%.*]] = lshr <3 x i42> [[SUB]], +; CHECK-NEXT: [[SHR:%.*]] = lshr <3 x i42> [[SUB]], ; CHECK-NEXT: ret <3 x i42> [[SHR]] ; %sub = sub nsw <3 x i42> %x, %y - %shr = lshr <3 x i42> %sub, + %shr = lshr <3 x i42> %sub, ret <3 x i42> %shr } @@ -572,14 +572,14 @@ define <3 x i43> @ashr_sub_nsw_splat(<3 x i43> %x, <3 x i43> %y) { ret <3 x i43> %shr } -define <3 x i43> @ashr_sub_nsw_splat_undef(<3 x i43> %x, <3 x i43> %y) { -; CHECK-LABEL: @ashr_sub_nsw_splat_undef( +define <3 x i43> @ashr_sub_nsw_splat_poison(<3 x i43> %x, <3 x i43> %y) { +; CHECK-LABEL: @ashr_sub_nsw_splat_poison( ; CHECK-NEXT: [[SUB:%.*]] = sub nsw <3 x i43> [[X:%.*]], [[Y:%.*]] -; CHECK-NEXT: [[SHR:%.*]] = ashr <3 x i43> [[SUB]], +; CHECK-NEXT: [[SHR:%.*]] = ashr <3 x i43> [[SUB]], ; CHECK-NEXT: ret <3 x i43> [[SHR]] ; %sub = sub nsw <3 x i43> %x, %y - %shr = ashr <3 x i43> %sub, + %shr = ashr <3 x i43> %sub, ret <3 x i43> %shr } diff --git a/llvm/test/Transforms/InstCombine/ashr-or-mul-abs.ll b/llvm/test/Transforms/InstCombine/ashr-or-mul-abs.ll index 3cf312e426ed..46a7f2f1189e 100644 --- a/llvm/test/Transforms/InstCombine/ashr-or-mul-abs.ll +++ b/llvm/test/Transforms/InstCombine/ashr-or-mul-abs.ll @@ -62,13 +62,13 @@ define <4 x i32> @ashr_or_mul_to_abs_vec2(<4 x i32> %X) { ret <4 x i32> %i2 } -define <4 x i32> @ashr_or_mul_to_abs_vec3_undef(<4 x i32> %X) { -; CHECK-LABEL: @ashr_or_mul_to_abs_vec3_undef( +define <4 x i32> @ashr_or_mul_to_abs_vec3_poison(<4 x i32> %X) { +; CHECK-LABEL: @ashr_or_mul_to_abs_vec3_poison( ; CHECK-NEXT: [[I2:%.*]] = call <4 x i32> @llvm.abs.v4i32(<4 x i32> [[X:%.*]], i1 false) ; CHECK-NEXT: ret <4 x i32> [[I2]] ; - %i = ashr <4 x i32> %X, - %i1 = or <4 x i32> %i, + %i = ashr <4 x i32> %X, + %i1 = or <4 x i32> %i, %i2 = mul <4 x i32> %i1, %X ret <4 x i32> %i2 } diff --git a/llvm/test/Transforms/InstCombine/binop-and-shifts.ll b/llvm/test/Transforms/InstCombine/binop-and-shifts.ll index 148963894b89..f776dc13bb4e 100644 --- a/llvm/test/Transforms/InstCombine/binop-and-shifts.ll +++ b/llvm/test/Transforms/InstCombine/binop-and-shifts.ll @@ -178,27 +178,27 @@ define <2 x i8> @shl_xor_and(<2 x i8> %x, <2 x i8> %y) { ; CHECK-LABEL: @shl_xor_and( ; CHECK-NEXT: [[TMP1:%.*]] = xor <2 x i8> [[Y:%.*]], ; CHECK-NEXT: [[TMP2:%.*]] = and <2 x i8> [[TMP1]], [[X:%.*]] -; CHECK-NEXT: [[BW1:%.*]] = shl <2 x i8> [[TMP2]], +; CHECK-NEXT: [[BW1:%.*]] = shl <2 x i8> [[TMP2]], ; CHECK-NEXT: ret <2 x i8> [[BW1]] ; - %shift1 = shl <2 x i8> %x, - %shift2 = shl <2 x i8> %y, - %bw2 = xor <2 x i8> %shift2, + %shift1 = shl <2 x i8> %x, + %shift2 = shl <2 x i8> %y, + %bw2 = xor <2 x i8> %shift2, %bw1 = and <2 x i8> %bw2, %shift1 ret <2 x i8> %bw1 } define <2 x i8> @shl_xor_and_fail(<2 x i8> %x, <2 x i8> %y) { ; CHECK-LABEL: @shl_xor_and_fail( -; CHECK-NEXT: [[SHIFT1:%.*]] = shl <2 x i8> [[X:%.*]], -; CHECK-NEXT: [[SHIFT2:%.*]] = shl <2 x i8> [[Y:%.*]], -; CHECK-NEXT: [[BW2:%.*]] = xor <2 x i8> [[SHIFT2]], +; CHECK-NEXT: [[SHIFT1:%.*]] = shl <2 x i8> [[X:%.*]], +; CHECK-NEXT: [[SHIFT2:%.*]] = shl <2 x i8> [[Y:%.*]], +; CHECK-NEXT: [[BW2:%.*]] = xor <2 x i8> [[SHIFT2]], ; CHECK-NEXT: [[BW1:%.*]] = and <2 x i8> [[SHIFT1]], [[BW2]] ; CHECK-NEXT: ret <2 x i8> [[BW1]] ; - %shift1 = shl <2 x i8> %x, - %shift2 = shl <2 x i8> %y, - %bw2 = xor <2 x i8> %shift2, + %shift1 = shl <2 x i8> %x, + %shift2 = shl <2 x i8> %y, + %bw2 = xor <2 x i8> %shift2, %bw1 = and <2 x i8> %shift1, %bw2 ret <2 x i8> %bw1 } @@ -321,13 +321,13 @@ define <2 x i8> @lshr_add_and(<2 x i8> %x, <2 x i8> %y) { define <2 x i8> @lshr_add_or_fail_dif_masks(<2 x i8> %x, <2 x i8> %y) { ; CHECK-LABEL: @lshr_add_or_fail_dif_masks( ; CHECK-NEXT: [[SHIFT1:%.*]] = lshr <2 x i8> [[X:%.*]], -; CHECK-NEXT: [[SHIFT2:%.*]] = lshr <2 x i8> [[Y:%.*]], -; CHECK-NEXT: [[BW2:%.*]] = add <2 x i8> [[SHIFT2]], +; CHECK-NEXT: [[SHIFT2:%.*]] = lshr <2 x i8> [[Y:%.*]], +; CHECK-NEXT: [[BW2:%.*]] = add nsw <2 x i8> [[SHIFT2]], ; CHECK-NEXT: [[BW1:%.*]] = and <2 x i8> [[SHIFT1]], [[BW2]] ; CHECK-NEXT: ret <2 x i8> [[BW1]] ; %shift1 = lshr <2 x i8> %x, - %shift2 = lshr <2 x i8> %y, + %shift2 = lshr <2 x i8> %y, %bw2 = add <2 x i8> %shift2, %bw1 = and <2 x i8> %shift1, %bw2 ret <2 x i8> %bw1 @@ -659,8 +659,8 @@ define <4 x i8> @and_ashr_not_vec_commuted(<4 x i8> %x, <4 x i8> %y, <4 x i8> %s ret <4 x i8> %and } -define <4 x i8> @and_ashr_not_vec_undef_1(<4 x i8> %x, <4 x i8> %y, <4 x i8> %shamt) { -; CHECK-LABEL: @and_ashr_not_vec_undef_1( +define <4 x i8> @and_ashr_not_vec_poison_1(<4 x i8> %x, <4 x i8> %y, <4 x i8> %shamt) { +; CHECK-LABEL: @and_ashr_not_vec_poison_1( ; CHECK-NEXT: [[TMP1:%.*]] = xor <4 x i8> [[Y:%.*]], ; CHECK-NEXT: [[TMP2:%.*]] = and <4 x i8> [[TMP1]], [[X:%.*]] ; CHECK-NEXT: [[AND:%.*]] = ashr <4 x i8> [[TMP2]], [[SHAMT:%.*]] @@ -668,18 +668,18 @@ define <4 x i8> @and_ashr_not_vec_undef_1(<4 x i8> %x, <4 x i8> %y, <4 x i8> %sh ; %x.shift = ashr <4 x i8> %x, %shamt %y.shift = ashr <4 x i8> %y, %shamt - %y.shift.not = xor <4 x i8> %y.shift, + %y.shift.not = xor <4 x i8> %y.shift, %and = and <4 x i8> %x.shift, %y.shift.not ret <4 x i8> %and } -define <4 x i8> @and_ashr_not_vec_undef_2(<4 x i8> %x, <4 x i8> %y, <4 x i8> %shamt) { -; CHECK-LABEL: @and_ashr_not_vec_undef_2( -; CHECK-NEXT: ret <4 x i8> zeroinitializer +define <4 x i8> @and_ashr_not_vec_poison_2(<4 x i8> %x, <4 x i8> %y, <4 x i8> %shamt) { +; CHECK-LABEL: @and_ashr_not_vec_poison_2( +; CHECK-NEXT: ret <4 x i8> poison ; %x.shift = ashr <4 x i8> %x, %shamt %y.shift = ashr <4 x i8> %y, %shamt - %y.shift.not = xor <4 x i8> %y.shift, + %y.shift.not = xor <4 x i8> %y.shift, %and = and <4 x i8> %x.shift, %y.shift.not ret <4 x i8> %and } @@ -793,8 +793,8 @@ define <4 x i8> @or_ashr_not_vec_commuted(<4 x i8> %x, <4 x i8> %y, <4 x i8> %sh ret <4 x i8> %or } -define <4 x i8> @or_ashr_not_vec_undef_1(<4 x i8> %x, <4 x i8> %y, <4 x i8> %shamt) { -; CHECK-LABEL: @or_ashr_not_vec_undef_1( +define <4 x i8> @or_ashr_not_vec_poison_1(<4 x i8> %x, <4 x i8> %y, <4 x i8> %shamt) { +; CHECK-LABEL: @or_ashr_not_vec_poison_1( ; CHECK-NEXT: [[TMP1:%.*]] = xor <4 x i8> [[Y:%.*]], ; CHECK-NEXT: [[TMP2:%.*]] = or <4 x i8> [[TMP1]], [[X:%.*]] ; CHECK-NEXT: [[OR:%.*]] = ashr <4 x i8> [[TMP2]], [[SHAMT:%.*]] @@ -802,18 +802,18 @@ define <4 x i8> @or_ashr_not_vec_undef_1(<4 x i8> %x, <4 x i8> %y, <4 x i8> %sha ; %x.shift = ashr <4 x i8> %x, %shamt %y.shift = ashr <4 x i8> %y, %shamt - %y.shift.not = xor <4 x i8> %y.shift, + %y.shift.not = xor <4 x i8> %y.shift, %or = or <4 x i8> %x.shift, %y.shift.not ret <4 x i8> %or } -define <4 x i8> @or_ashr_not_vec_undef_2(<4 x i8> %x, <4 x i8> %y, <4 x i8> %shamt) { -; CHECK-LABEL: @or_ashr_not_vec_undef_2( -; CHECK-NEXT: ret <4 x i8> +define <4 x i8> @or_ashr_not_vec_poison_2(<4 x i8> %x, <4 x i8> %y, <4 x i8> %shamt) { +; CHECK-LABEL: @or_ashr_not_vec_poison_2( +; CHECK-NEXT: ret <4 x i8> poison ; %x.shift = ashr <4 x i8> %x, %shamt %y.shift = ashr <4 x i8> %y, %shamt - %y.shift.not = xor <4 x i8> %y.shift, + %y.shift.not = xor <4 x i8> %y.shift, %or = or <4 x i8> %x.shift, %y.shift.not ret <4 x i8> %or } @@ -926,8 +926,8 @@ define <4 x i8> @xor_ashr_not_vec_commuted(<4 x i8> %x, <4 x i8> %y, <4 x i8> %s ret <4 x i8> %xor } -define <4 x i8> @xor_ashr_not_vec_undef_1(<4 x i8> %x, <4 x i8> %y, <4 x i8> %shamt) { -; CHECK-LABEL: @xor_ashr_not_vec_undef_1( +define <4 x i8> @xor_ashr_not_vec_poison_1(<4 x i8> %x, <4 x i8> %y, <4 x i8> %shamt) { +; CHECK-LABEL: @xor_ashr_not_vec_poison_1( ; CHECK-NEXT: [[TMP1:%.*]] = xor <4 x i8> [[Y:%.*]], [[X:%.*]] ; CHECK-NEXT: [[DOTNOT:%.*]] = ashr <4 x i8> [[TMP1]], [[SHAMT:%.*]] ; CHECK-NEXT: [[XOR:%.*]] = xor <4 x i8> [[DOTNOT]], @@ -935,18 +935,18 @@ define <4 x i8> @xor_ashr_not_vec_undef_1(<4 x i8> %x, <4 x i8> %y, <4 x i8> %sh ; %x.shift = ashr <4 x i8> %x, %shamt %y.shift = ashr <4 x i8> %y, %shamt - %y.shift.not = xor <4 x i8> %y.shift, + %y.shift.not = xor <4 x i8> %y.shift, %xor = xor <4 x i8> %x.shift, %y.shift.not ret <4 x i8> %xor } -define <4 x i8> @xor_ashr_not_vec_undef_2(<4 x i8> %x, <4 x i8> %y, <4 x i8> %shamt) { -; CHECK-LABEL: @xor_ashr_not_vec_undef_2( -; CHECK-NEXT: ret <4 x i8> undef +define <4 x i8> @xor_ashr_not_vec_poison_2(<4 x i8> %x, <4 x i8> %y, <4 x i8> %shamt) { +; CHECK-LABEL: @xor_ashr_not_vec_poison_2( +; CHECK-NEXT: ret <4 x i8> poison ; %x.shift = ashr <4 x i8> %x, %shamt %y.shift = ashr <4 x i8> %y, %shamt - %y.shift.not = xor <4 x i8> %y.shift, + %y.shift.not = xor <4 x i8> %y.shift, %xor = xor <4 x i8> %x.shift, %y.shift.not ret <4 x i8> %xor } diff --git a/llvm/test/Transforms/InstCombine/binop-of-displaced-shifts.ll b/llvm/test/Transforms/InstCombine/binop-of-displaced-shifts.ll index 27a3c8743368..a16ad4ddb806 100644 --- a/llvm/test/Transforms/InstCombine/binop-of-displaced-shifts.ll +++ b/llvm/test/Transforms/InstCombine/binop-of-displaced-shifts.ll @@ -202,41 +202,41 @@ define <2 x i8> @shl_or_non_splat(<2 x i8> %x) { ret <2 x i8> %binop } -define <2 x i8> @shl_or_undef_in_add(<2 x i8> %x) { -; CHECK-LABEL: define <2 x i8> @shl_or_undef_in_add +define <2 x i8> @shl_or_poison_in_add(<2 x i8> %x) { +; CHECK-LABEL: define <2 x i8> @shl_or_poison_in_add ; CHECK-SAME: (<2 x i8> [[X:%.*]]) { ; CHECK-NEXT: [[BINOP:%.*]] = shl <2 x i8> , [[X]] ; CHECK-NEXT: ret <2 x i8> [[BINOP]] ; %shift = shl <2 x i8> , %x - %add = add <2 x i8> %x, + %add = add <2 x i8> %x, %shift2 = shl <2 x i8> , %add %binop = or <2 x i8> %shift, %shift2 ret <2 x i8> %binop } -define <2 x i8> @shl_or_undef_in_shift1(<2 x i8> %x) { -; CHECK-LABEL: define <2 x i8> @shl_or_undef_in_shift1 +define <2 x i8> @shl_or_poison_in_shift1(<2 x i8> %x) { +; CHECK-LABEL: define <2 x i8> @shl_or_poison_in_shift1 ; CHECK-SAME: (<2 x i8> [[X:%.*]]) { -; CHECK-NEXT: [[BINOP:%.*]] = shl <2 x i8> , [[X]] +; CHECK-NEXT: [[BINOP:%.*]] = shl <2 x i8> , [[X]] ; CHECK-NEXT: ret <2 x i8> [[BINOP]] ; - %shift = shl <2 x i8> , %x + %shift = shl <2 x i8> , %x %add = add <2 x i8> %x, %shift2 = shl <2 x i8> , %add %binop = or <2 x i8> %shift, %shift2 ret <2 x i8> %binop } -define <2 x i8> @shl_or_undef_in_shift2(<2 x i8> %x) { -; CHECK-LABEL: define <2 x i8> @shl_or_undef_in_shift2 +define <2 x i8> @shl_or_poison_in_shift2(<2 x i8> %x) { +; CHECK-LABEL: define <2 x i8> @shl_or_poison_in_shift2 ; CHECK-SAME: (<2 x i8> [[X:%.*]]) { -; CHECK-NEXT: [[BINOP:%.*]] = shl <2 x i8> , [[X]] +; CHECK-NEXT: [[BINOP:%.*]] = shl <2 x i8> , [[X]] ; CHECK-NEXT: ret <2 x i8> [[BINOP]] ; %shift = shl <2 x i8> , %x %add = add <2 x i8> %x, - %shift2 = shl <2 x i8> , %add + %shift2 = shl <2 x i8> , %add %binop = or <2 x i8> %shift, %shift2 ret <2 x i8> %binop } diff --git a/llvm/test/Transforms/InstCombine/canonicalize-clamp-like-pattern-between-zero-and-positive-threshold.ll b/llvm/test/Transforms/InstCombine/canonicalize-clamp-like-pattern-between-zero-and-positive-threshold.ll index 4547008b7609..c555970ea434 100644 --- a/llvm/test/Transforms/InstCombine/canonicalize-clamp-like-pattern-between-zero-and-positive-threshold.ll +++ b/llvm/test/Transforms/InstCombine/canonicalize-clamp-like-pattern-between-zero-and-positive-threshold.ll @@ -338,22 +338,22 @@ define <2 x i32> @t18_ult_slt_vec_nonsplat(<2 x i32> %x, <2 x i32> %replacement_ ret <2 x i32> %r } -define <3 x i32> @t19_ult_slt_vec_undef0(<3 x i32> %x, <3 x i32> %replacement_low, <3 x i32> %replacement_high) { -; CHECK-LABEL: @t19_ult_slt_vec_undef0( +define <3 x i32> @t19_ult_slt_vec_poison0(<3 x i32> %x, <3 x i32> %replacement_low, <3 x i32> %replacement_high) { +; CHECK-LABEL: @t19_ult_slt_vec_poison0( ; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <3 x i32> [[X:%.*]], zeroinitializer ; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <3 x i32> [[X]], ; CHECK-NEXT: [[TMP3:%.*]] = select <3 x i1> [[TMP1]], <3 x i32> [[REPLACEMENT_LOW:%.*]], <3 x i32> [[X]] ; CHECK-NEXT: [[R:%.*]] = select <3 x i1> [[TMP2]], <3 x i32> [[REPLACEMENT_HIGH:%.*]], <3 x i32> [[TMP3]] ; CHECK-NEXT: ret <3 x i32> [[R]] ; - %t0 = icmp slt <3 x i32> %x, + %t0 = icmp slt <3 x i32> %x, %t1 = select <3 x i1> %t0, <3 x i32> %replacement_low, <3 x i32> %replacement_high %t2 = icmp ult <3 x i32> %x, %r = select <3 x i1> %t2, <3 x i32> %x, <3 x i32> %t1 ret <3 x i32> %r } -define <3 x i32> @t20_ult_slt_vec_undef1(<3 x i32> %x, <3 x i32> %replacement_low, <3 x i32> %replacement_high) { -; CHECK-LABEL: @t20_ult_slt_vec_undef1( +define <3 x i32> @t20_ult_slt_vec_poison1(<3 x i32> %x, <3 x i32> %replacement_low, <3 x i32> %replacement_high) { +; CHECK-LABEL: @t20_ult_slt_vec_poison1( ; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <3 x i32> [[X:%.*]], zeroinitializer ; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <3 x i32> [[X]], ; CHECK-NEXT: [[TMP3:%.*]] = select <3 x i1> [[TMP1]], <3 x i32> [[REPLACEMENT_LOW:%.*]], <3 x i32> [[X]] @@ -362,21 +362,21 @@ define <3 x i32> @t20_ult_slt_vec_undef1(<3 x i32> %x, <3 x i32> %replacement_lo ; %t0 = icmp slt <3 x i32> %x, %t1 = select <3 x i1> %t0, <3 x i32> %replacement_low, <3 x i32> %replacement_high - %t2 = icmp ult <3 x i32> %x, + %t2 = icmp ult <3 x i32> %x, %r = select <3 x i1> %t2, <3 x i32> %x, <3 x i32> %t1 ret <3 x i32> %r } -define <3 x i32> @t21_ult_slt_vec_undef2(<3 x i32> %x, <3 x i32> %replacement_low, <3 x i32> %replacement_high) { -; CHECK-LABEL: @t21_ult_slt_vec_undef2( +define <3 x i32> @t21_ult_slt_vec_poison2(<3 x i32> %x, <3 x i32> %replacement_low, <3 x i32> %replacement_high) { +; CHECK-LABEL: @t21_ult_slt_vec_poison2( ; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <3 x i32> [[X:%.*]], zeroinitializer ; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <3 x i32> [[X]], ; CHECK-NEXT: [[TMP3:%.*]] = select <3 x i1> [[TMP1]], <3 x i32> [[REPLACEMENT_LOW:%.*]], <3 x i32> [[X]] ; CHECK-NEXT: [[R:%.*]] = select <3 x i1> [[TMP2]], <3 x i32> [[REPLACEMENT_HIGH:%.*]], <3 x i32> [[TMP3]] ; CHECK-NEXT: ret <3 x i32> [[R]] ; - %t0 = icmp slt <3 x i32> %x, + %t0 = icmp slt <3 x i32> %x, %t1 = select <3 x i1> %t0, <3 x i32> %replacement_low, <3 x i32> %replacement_high - %t2 = icmp ult <3 x i32> %x, + %t2 = icmp ult <3 x i32> %x, %r = select <3 x i1> %t2, <3 x i32> %x, <3 x i32> %t1 ret <3 x i32> %r } diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-eq-to-icmp-ule.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-eq-to-icmp-ule.ll index 5b7a99d53c30..759770688cf2 100644 --- a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-eq-to-icmp-ule.ll +++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-eq-to-icmp-ule.ll @@ -79,12 +79,12 @@ define <2 x i1> @p2_vec_nonsplat_edgecase1(<2 x i8> %x) { ret <2 x i1> %ret } -define <3 x i1> @p3_vec_splat_undef(<3 x i8> %x) { -; CHECK-LABEL: @p3_vec_splat_undef( +define <3 x i1> @p3_vec_splat_poison(<3 x i8> %x) { +; CHECK-LABEL: @p3_vec_splat_poison( ; CHECK-NEXT: [[RET:%.*]] = icmp ult <3 x i8> [[X:%.*]], ; CHECK-NEXT: ret <3 x i1> [[RET]] ; - %tmp0 = and <3 x i8> %x, + %tmp0 = and <3 x i8> %x, %ret = icmp eq <3 x i8> %tmp0, %x ret <3 x i1> %ret } diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ne-to-icmp-ugt.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ne-to-icmp-ugt.ll index 160d968b9ac4..95e6d5ac6a5f 100644 --- a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ne-to-icmp-ugt.ll +++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ne-to-icmp-ugt.ll @@ -79,22 +79,22 @@ define <2 x i1> @p2_vec_nonsplat_edgecase1(<2 x i8> %x) { ret <2 x i1> %ret } -define <3 x i1> @p3_vec_splat_undef(<3 x i8> %x) { -; CHECK-LABEL: @p3_vec_splat_undef( +define <3 x i1> @p3_vec_splat_poison(<3 x i8> %x) { +; CHECK-LABEL: @p3_vec_splat_poison( ; CHECK-NEXT: [[RET:%.*]] = icmp ugt <3 x i8> [[X:%.*]], ; CHECK-NEXT: ret <3 x i1> [[RET]] ; - %tmp0 = and <3 x i8> %x, + %tmp0 = and <3 x i8> %x, %ret = icmp ne <3 x i8> %tmp0, %x ret <3 x i1> %ret } -define <3 x i1> @p3_vec_nonsplat_undef(<3 x i8> %x) { -; CHECK-LABEL: @p3_vec_nonsplat_undef( +define <3 x i1> @p3_vec_nonsplat_poison(<3 x i8> %x) { +; CHECK-LABEL: @p3_vec_nonsplat_poison( ; CHECK-NEXT: [[RET:%.*]] = icmp ugt <3 x i8> [[X:%.*]], ; CHECK-NEXT: ret <3 x i1> [[RET]] ; - %tmp0 = and <3 x i8> %x, + %tmp0 = and <3 x i8> %x, %ret = icmp ne <3 x i8> %tmp0, %x ret <3 x i1> %ret } diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sge-to-icmp-sle.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sge-to-icmp-sle.ll index 60921042d524..ae503bfb1cfe 100644 --- a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sge-to-icmp-sle.ll +++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sge-to-icmp-sle.ll @@ -58,12 +58,12 @@ define <2 x i1> @p2_vec_nonsplat_edgecase(<2 x i8> %x) { ret <2 x i1> %ret } -define <3 x i1> @p3_vec_splat_undef(<3 x i8> %x) { -; CHECK-LABEL: @p3_vec_splat_undef( +define <3 x i1> @p3_vec_splat_poison(<3 x i8> %x) { +; CHECK-LABEL: @p3_vec_splat_poison( ; CHECK-NEXT: [[RET:%.*]] = icmp slt <3 x i8> [[X:%.*]], ; CHECK-NEXT: ret <3 x i1> [[RET]] ; - %tmp0 = and <3 x i8> %x, + %tmp0 = and <3 x i8> %x, %ret = icmp sge <3 x i8> %tmp0, %x ret <3 x i1> %ret } @@ -175,11 +175,11 @@ define <2 x i1> @n3_vec(<2 x i8> %x) { define <3 x i1> @n4_vec(<3 x i8> %x) { ; CHECK-LABEL: @n4_vec( -; CHECK-NEXT: [[TMP0:%.*]] = and <3 x i8> [[X:%.*]], +; CHECK-NEXT: [[TMP0:%.*]] = and <3 x i8> [[X:%.*]], ; CHECK-NEXT: [[RET:%.*]] = icmp sge <3 x i8> [[TMP0]], [[X]] ; CHECK-NEXT: ret <3 x i1> [[RET]] ; - %tmp0 = and <3 x i8> %x, + %tmp0 = and <3 x i8> %x, %ret = icmp sge <3 x i8> %tmp0, %x ret <3 x i1> %ret } diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sgt-to-icmp-sgt.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sgt-to-icmp-sgt.ll index 6345e70d7220..f1333fed2c51 100644 --- a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sgt-to-icmp-sgt.ll +++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sgt-to-icmp-sgt.ll @@ -72,26 +72,26 @@ define <2 x i1> @p2_vec_nonsplat_edgecase() { ret <2 x i1> %ret } -define <3 x i1> @p3_vec_splat_undef() { -; CHECK-LABEL: @p3_vec_splat_undef( +define <3 x i1> @p3_vec_splat_poison() { +; CHECK-LABEL: @p3_vec_splat_poison( ; CHECK-NEXT: [[X:%.*]] = call <3 x i8> @gen3x8() ; CHECK-NEXT: [[RET:%.*]] = icmp sgt <3 x i8> [[X]], ; CHECK-NEXT: ret <3 x i1> [[RET]] ; %x = call <3 x i8> @gen3x8() - %tmp0 = and <3 x i8> %x, + %tmp0 = and <3 x i8> %x, %ret = icmp sgt <3 x i8> %x, %tmp0 ret <3 x i1> %ret } -define <3 x i1> @p3_vec_nonsplat_undef() { -; CHECK-LABEL: @p3_vec_nonsplat_undef( +define <3 x i1> @p3_vec_nonsplat_poison() { +; CHECK-LABEL: @p3_vec_nonsplat_poison( ; CHECK-NEXT: [[X:%.*]] = call <3 x i8> @gen3x8() ; CHECK-NEXT: [[RET:%.*]] = icmp sgt <3 x i8> [[X]], ; CHECK-NEXT: ret <3 x i1> [[RET]] ; %x = call <3 x i8> @gen3x8() - %tmp0 = and <3 x i8> %x, + %tmp0 = and <3 x i8> %x, %ret = icmp sgt <3 x i8> %x, %tmp0 ret <3 x i1> %ret } @@ -212,12 +212,12 @@ define <2 x i1> @n3_vec() { define <3 x i1> @n4_vec() { ; CHECK-LABEL: @n4_vec( ; CHECK-NEXT: [[X:%.*]] = call <3 x i8> @gen3x8() -; CHECK-NEXT: [[TMP0:%.*]] = and <3 x i8> [[X]], +; CHECK-NEXT: [[TMP0:%.*]] = and <3 x i8> [[X]], ; CHECK-NEXT: [[RET:%.*]] = icmp sgt <3 x i8> [[X]], [[TMP0]] ; CHECK-NEXT: ret <3 x i1> [[RET]] ; %x = call <3 x i8> @gen3x8() - %tmp0 = and <3 x i8> %x, + %tmp0 = and <3 x i8> %x, %ret = icmp sgt <3 x i8> %x, %tmp0 ret <3 x i1> %ret } diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sle-to-icmp-sle.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sle-to-icmp-sle.ll index b7aec53fed67..4bed21a525f0 100644 --- a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sle-to-icmp-sle.ll +++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sle-to-icmp-sle.ll @@ -72,14 +72,14 @@ define <2 x i1> @p2_vec_nonsplat_edgecase() { ret <2 x i1> %ret } -define <3 x i1> @p3_vec_splat_undef() { -; CHECK-LABEL: @p3_vec_splat_undef( +define <3 x i1> @p3_vec_splat_poison() { +; CHECK-LABEL: @p3_vec_splat_poison( ; CHECK-NEXT: [[X:%.*]] = call <3 x i8> @gen3x8() ; CHECK-NEXT: [[RET:%.*]] = icmp slt <3 x i8> [[X]], ; CHECK-NEXT: ret <3 x i1> [[RET]] ; %x = call <3 x i8> @gen3x8() - %tmp0 = and <3 x i8> %x, + %tmp0 = and <3 x i8> %x, %ret = icmp sle <3 x i8> %x, %tmp0 ret <3 x i1> %ret } @@ -200,12 +200,12 @@ define <2 x i1> @n3_vec() { define <3 x i1> @n4_vec() { ; CHECK-LABEL: @n4_vec( ; CHECK-NEXT: [[X:%.*]] = call <3 x i8> @gen3x8() -; CHECK-NEXT: [[TMP0:%.*]] = and <3 x i8> [[X]], +; CHECK-NEXT: [[TMP0:%.*]] = and <3 x i8> [[X]], ; CHECK-NEXT: [[RET:%.*]] = icmp sle <3 x i8> [[X]], [[TMP0]] ; CHECK-NEXT: ret <3 x i1> [[RET]] ; %x = call <3 x i8> @gen3x8() - %tmp0 = and <3 x i8> %x, + %tmp0 = and <3 x i8> %x, %ret = icmp sle <3 x i8> %x, %tmp0 ret <3 x i1> %ret } diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-slt-to-icmp-sgt.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-slt-to-icmp-sgt.ll index 56661d335c4f..be6e3d0306bc 100644 --- a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-slt-to-icmp-sgt.ll +++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-slt-to-icmp-sgt.ll @@ -58,22 +58,22 @@ define <2 x i1> @p2_vec_nonsplat_edgecase(<2 x i8> %x) { ret <2 x i1> %ret } -define <3 x i1> @p3_vec_splat_undef(<3 x i8> %x) { -; CHECK-LABEL: @p3_vec_splat_undef( +define <3 x i1> @p3_vec_splat_poison(<3 x i8> %x) { +; CHECK-LABEL: @p3_vec_splat_poison( ; CHECK-NEXT: [[RET:%.*]] = icmp sgt <3 x i8> [[X:%.*]], ; CHECK-NEXT: ret <3 x i1> [[RET]] ; - %tmp0 = and <3 x i8> %x, + %tmp0 = and <3 x i8> %x, %ret = icmp slt <3 x i8> %tmp0, %x ret <3 x i1> %ret } -define <3 x i1> @p3_vec_nonsplat_undef(<3 x i8> %x) { -; CHECK-LABEL: @p3_vec_nonsplat_undef( +define <3 x i1> @p3_vec_nonsplat_poison(<3 x i8> %x) { +; CHECK-LABEL: @p3_vec_nonsplat_poison( ; CHECK-NEXT: [[RET:%.*]] = icmp sgt <3 x i8> [[X:%.*]], ; CHECK-NEXT: ret <3 x i1> [[RET]] ; - %tmp0 = and <3 x i8> %x, + %tmp0 = and <3 x i8> %x, %ret = icmp slt <3 x i8> %tmp0, %x ret <3 x i1> %ret } @@ -185,11 +185,11 @@ define <2 x i1> @n3(<2 x i8> %x) { define <3 x i1> @n4(<3 x i8> %x) { ; CHECK-LABEL: @n4( -; CHECK-NEXT: [[TMP0:%.*]] = and <3 x i8> [[X:%.*]], +; CHECK-NEXT: [[TMP0:%.*]] = and <3 x i8> [[X:%.*]], ; CHECK-NEXT: [[RET:%.*]] = icmp slt <3 x i8> [[TMP0]], [[X]] ; CHECK-NEXT: ret <3 x i1> [[RET]] ; - %tmp0 = and <3 x i8> %x, + %tmp0 = and <3 x i8> %x, %ret = icmp slt <3 x i8> %tmp0, %x ret <3 x i1> %ret } diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-uge-to-icmp-ule.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-uge-to-icmp-ule.ll index a93e8f779435..cfd48821b2c1 100644 --- a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-uge-to-icmp-ule.ll +++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-uge-to-icmp-ule.ll @@ -79,12 +79,12 @@ define <2 x i1> @p2_vec_nonsplat_edgecase1(<2 x i8> %x) { ret <2 x i1> %ret } -define <3 x i1> @p3_vec_splat_undef(<3 x i8> %x) { -; CHECK-LABEL: @p3_vec_splat_undef( +define <3 x i1> @p3_vec_splat_poison(<3 x i8> %x) { +; CHECK-LABEL: @p3_vec_splat_poison( ; CHECK-NEXT: [[RET:%.*]] = icmp ult <3 x i8> [[X:%.*]], ; CHECK-NEXT: ret <3 x i1> [[RET]] ; - %tmp0 = and <3 x i8> %x, + %tmp0 = and <3 x i8> %x, %ret = icmp uge <3 x i8> %tmp0, %x ret <3 x i1> %ret } diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ugt-to-icmp-ugt.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ugt-to-icmp-ugt.ll index 73ea4d456d24..6f6ba95a81c7 100644 --- a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ugt-to-icmp-ugt.ll +++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ugt-to-icmp-ugt.ll @@ -95,26 +95,26 @@ define <2 x i1> @p2_vec_nonsplat_edgecase1() { ret <2 x i1> %ret } -define <3 x i1> @p3_vec_splat_undef() { -; CHECK-LABEL: @p3_vec_splat_undef( +define <3 x i1> @p3_vec_splat_poison() { +; CHECK-LABEL: @p3_vec_splat_poison( ; CHECK-NEXT: [[X:%.*]] = call <3 x i8> @gen3x8() ; CHECK-NEXT: [[RET:%.*]] = icmp ugt <3 x i8> [[X]], ; CHECK-NEXT: ret <3 x i1> [[RET]] ; %x = call <3 x i8> @gen3x8() - %tmp0 = and <3 x i8> %x, + %tmp0 = and <3 x i8> %x, %ret = icmp ugt <3 x i8> %x, %tmp0 ret <3 x i1> %ret } -define <3 x i1> @p3_vec_nonsplat_undef() { -; CHECK-LABEL: @p3_vec_nonsplat_undef( +define <3 x i1> @p3_vec_nonsplat_poison() { +; CHECK-LABEL: @p3_vec_nonsplat_poison( ; CHECK-NEXT: [[X:%.*]] = call <3 x i8> @gen3x8() ; CHECK-NEXT: [[RET:%.*]] = icmp ugt <3 x i8> [[X]], ; CHECK-NEXT: ret <3 x i1> [[RET]] ; %x = call <3 x i8> @gen3x8() - %tmp0 = and <3 x i8> %x, + %tmp0 = and <3 x i8> %x, %ret = icmp ugt <3 x i8> %x, %tmp0 ret <3 x i1> %ret } diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ule-to-icmp-ule.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ule-to-icmp-ule.ll index 53886b5f2dc9..54f00321c4cf 100644 --- a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ule-to-icmp-ule.ll +++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ule-to-icmp-ule.ll @@ -95,14 +95,14 @@ define <2 x i1> @p2_vec_nonsplat_edgecase1() { ret <2 x i1> %ret } -define <3 x i1> @p3_vec_splat_undef() { -; CHECK-LABEL: @p3_vec_splat_undef( +define <3 x i1> @p3_vec_splat_poison() { +; CHECK-LABEL: @p3_vec_splat_poison( ; CHECK-NEXT: [[X:%.*]] = call <3 x i8> @gen3x8() ; CHECK-NEXT: [[RET:%.*]] = icmp ult <3 x i8> [[X]], ; CHECK-NEXT: ret <3 x i1> [[RET]] ; %x = call <3 x i8> @gen3x8() - %tmp0 = and <3 x i8> %x, + %tmp0 = and <3 x i8> %x, %ret = icmp ule <3 x i8> %x, %tmp0 ret <3 x i1> %ret } diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ult-to-icmp-ugt.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ult-to-icmp-ugt.ll index d66be571008c..008fc6d2d6ed 100644 --- a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ult-to-icmp-ugt.ll +++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ult-to-icmp-ugt.ll @@ -80,22 +80,22 @@ define <2 x i1> @p2_vec_nonsplat_edgecase1(<2 x i8> %x) { ret <2 x i1> %ret } -define <3 x i1> @p3_vec_splat_undef(<3 x i8> %x) { -; CHECK-LABEL: @p3_vec_splat_undef( +define <3 x i1> @p3_vec_splat_poison(<3 x i8> %x) { +; CHECK-LABEL: @p3_vec_splat_poison( ; CHECK-NEXT: [[RET:%.*]] = icmp ugt <3 x i8> [[X:%.*]], ; CHECK-NEXT: ret <3 x i1> [[RET]] ; - %tmp0 = and <3 x i8> %x, + %tmp0 = and <3 x i8> %x, %ret = icmp ult <3 x i8> %tmp0, %x ret <3 x i1> %ret } -define <3 x i1> @p3_vec_nonsplat_undef(<3 x i8> %x) { -; CHECK-LABEL: @p3_vec_nonsplat_undef( +define <3 x i1> @p3_vec_nonsplat_poison(<3 x i8> %x) { +; CHECK-LABEL: @p3_vec_nonsplat_poison( ; CHECK-NEXT: [[RET:%.*]] = icmp ugt <3 x i8> [[X:%.*]], ; CHECK-NEXT: ret <3 x i1> [[RET]] ; - %tmp0 = and <3 x i8> %x, + %tmp0 = and <3 x i8> %x, %ret = icmp ult <3 x i8> %tmp0, %x ret <3 x i1> %ret } diff --git a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-and-icmp-eq-to-icmp-ule.ll b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-and-icmp-eq-to-icmp-ule.ll index 38611d8b53a9..dc5658d302d9 100644 --- a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-and-icmp-eq-to-icmp-ule.ll +++ b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-and-icmp-eq-to-icmp-ule.ll @@ -40,13 +40,13 @@ define <2 x i1> @p1_vec(<2 x i8> %x, <2 x i8> %y) { ret <2 x i1> %ret } -define <3 x i1> @p2_vec_undef(<3 x i8> %x, <3 x i8> %y) { -; CHECK-LABEL: @p2_vec_undef( -; CHECK-NEXT: [[TMP0:%.*]] = lshr <3 x i8> , [[Y:%.*]] +define <3 x i1> @p2_vec_poison(<3 x i8> %x, <3 x i8> %y) { +; CHECK-LABEL: @p2_vec_poison( +; CHECK-NEXT: [[TMP0:%.*]] = lshr <3 x i8> , [[Y:%.*]] ; CHECK-NEXT: [[RET:%.*]] = icmp uge <3 x i8> [[TMP0]], [[X:%.*]] ; CHECK-NEXT: ret <3 x i1> [[RET]] ; - %tmp0 = lshr <3 x i8> , %y + %tmp0 = lshr <3 x i8> , %y %tmp1 = and <3 x i8> %tmp0, %x %ret = icmp eq <3 x i8> %tmp1, %x ret <3 x i1> %ret diff --git a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-and-icmp-ne-to-icmp-ugt.ll b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-and-icmp-ne-to-icmp-ugt.ll index 37d317b695f6..8fbbd2bb9907 100644 --- a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-and-icmp-ne-to-icmp-ugt.ll +++ b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-and-icmp-ne-to-icmp-ugt.ll @@ -40,13 +40,13 @@ define <2 x i1> @p1_vec(<2 x i8> %x, <2 x i8> %y) { ret <2 x i1> %ret } -define <3 x i1> @p2_vec_undef(<3 x i8> %x, <3 x i8> %y) { -; CHECK-LABEL: @p2_vec_undef( -; CHECK-NEXT: [[TMP0:%.*]] = lshr <3 x i8> , [[Y:%.*]] +define <3 x i1> @p2_vec_poison(<3 x i8> %x, <3 x i8> %y) { +; CHECK-LABEL: @p2_vec_poison( +; CHECK-NEXT: [[TMP0:%.*]] = lshr <3 x i8> , [[Y:%.*]] ; CHECK-NEXT: [[RET:%.*]] = icmp ult <3 x i8> [[TMP0]], [[X:%.*]] ; CHECK-NEXT: ret <3 x i1> [[RET]] ; - %tmp0 = lshr <3 x i8> , %y + %tmp0 = lshr <3 x i8> , %y %tmp1 = and <3 x i8> %tmp0, %x %ret = icmp ne <3 x i8> %tmp1, %x ret <3 x i1> %ret diff --git a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v2-and-icmp-eq-to-icmp-ule.ll b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v2-and-icmp-eq-to-icmp-ule.ll index dfd67eae8aaf..88487b38e2c7 100644 --- a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v2-and-icmp-eq-to-icmp-ule.ll +++ b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v2-and-icmp-eq-to-icmp-ule.ll @@ -44,40 +44,40 @@ define <2 x i1> @p1_vec(<2 x i8> %x, <2 x i8> %y) { ret <2 x i1> %ret } -define <3 x i1> @p2_vec_undef0(<3 x i8> %x, <3 x i8> %y) { -; CHECK-LABEL: @p2_vec_undef0( +define <3 x i1> @p2_vec_poison0(<3 x i8> %x, <3 x i8> %y) { +; CHECK-LABEL: @p2_vec_poison0( ; CHECK-NEXT: [[X_HIGHBITS:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]] ; CHECK-NEXT: [[RET:%.*]] = icmp eq <3 x i8> [[X_HIGHBITS]], zeroinitializer ; CHECK-NEXT: ret <3 x i1> [[RET]] ; - %t0 = shl <3 x i8> , %y + %t0 = shl <3 x i8> , %y %t1 = xor <3 x i8> %t0, %t2 = and <3 x i8> %t1, %x %ret = icmp eq <3 x i8> %t2, %x ret <3 x i1> %ret } -define <3 x i1> @p3_vec_undef0(<3 x i8> %x, <3 x i8> %y) { -; CHECK-LABEL: @p3_vec_undef0( +define <3 x i1> @p3_vec_poison0(<3 x i8> %x, <3 x i8> %y) { +; CHECK-LABEL: @p3_vec_poison0( ; CHECK-NEXT: [[X_HIGHBITS:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]] ; CHECK-NEXT: [[RET:%.*]] = icmp eq <3 x i8> [[X_HIGHBITS]], zeroinitializer ; CHECK-NEXT: ret <3 x i1> [[RET]] ; %t0 = shl <3 x i8> , %y - %t1 = xor <3 x i8> %t0, + %t1 = xor <3 x i8> %t0, %t2 = and <3 x i8> %t1, %x %ret = icmp eq <3 x i8> %t2, %x ret <3 x i1> %ret } -define <3 x i1> @p4_vec_undef2(<3 x i8> %x, <3 x i8> %y) { -; CHECK-LABEL: @p4_vec_undef2( +define <3 x i1> @p4_vec_poison2(<3 x i8> %x, <3 x i8> %y) { +; CHECK-LABEL: @p4_vec_poison2( ; CHECK-NEXT: [[X_HIGHBITS:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]] ; CHECK-NEXT: [[RET:%.*]] = icmp eq <3 x i8> [[X_HIGHBITS]], zeroinitializer ; CHECK-NEXT: ret <3 x i1> [[RET]] ; - %t0 = shl <3 x i8> , %y - %t1 = xor <3 x i8> %t0, + %t0 = shl <3 x i8> , %y + %t1 = xor <3 x i8> %t0, %t2 = and <3 x i8> %t1, %x %ret = icmp eq <3 x i8> %t2, %x ret <3 x i1> %ret diff --git a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v2-and-icmp-ne-to-icmp-ugt.ll b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v2-and-icmp-ne-to-icmp-ugt.ll index 608e133ec7f7..b717925fd644 100644 --- a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v2-and-icmp-ne-to-icmp-ugt.ll +++ b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v2-and-icmp-ne-to-icmp-ugt.ll @@ -44,40 +44,40 @@ define <2 x i1> @p1_vec(<2 x i8> %x, <2 x i8> %y) { ret <2 x i1> %ret } -define <3 x i1> @p2_vec_undef0(<3 x i8> %x, <3 x i8> %y) { -; CHECK-LABEL: @p2_vec_undef0( +define <3 x i1> @p2_vec_poison0(<3 x i8> %x, <3 x i8> %y) { +; CHECK-LABEL: @p2_vec_poison0( ; CHECK-NEXT: [[X_HIGHBITS:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]] ; CHECK-NEXT: [[RET:%.*]] = icmp ne <3 x i8> [[X_HIGHBITS]], zeroinitializer ; CHECK-NEXT: ret <3 x i1> [[RET]] ; - %t0 = shl <3 x i8> , %y + %t0 = shl <3 x i8> , %y %t1 = xor <3 x i8> %t0, %t2 = and <3 x i8> %t1, %x %ret = icmp ne <3 x i8> %t2, %x ret <3 x i1> %ret } -define <3 x i1> @p3_vec_undef0(<3 x i8> %x, <3 x i8> %y) { -; CHECK-LABEL: @p3_vec_undef0( +define <3 x i1> @p3_vec_poison0(<3 x i8> %x, <3 x i8> %y) { +; CHECK-LABEL: @p3_vec_poison0( ; CHECK-NEXT: [[X_HIGHBITS:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]] ; CHECK-NEXT: [[RET:%.*]] = icmp ne <3 x i8> [[X_HIGHBITS]], zeroinitializer ; CHECK-NEXT: ret <3 x i1> [[RET]] ; %t0 = shl <3 x i8> , %y - %t1 = xor <3 x i8> %t0, + %t1 = xor <3 x i8> %t0, %t2 = and <3 x i8> %t1, %x %ret = icmp ne <3 x i8> %t2, %x ret <3 x i1> %ret } -define <3 x i1> @p4_vec_undef2(<3 x i8> %x, <3 x i8> %y) { -; CHECK-LABEL: @p4_vec_undef2( +define <3 x i1> @p4_vec_poison2(<3 x i8> %x, <3 x i8> %y) { +; CHECK-LABEL: @p4_vec_poison2( ; CHECK-NEXT: [[X_HIGHBITS:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]] ; CHECK-NEXT: [[RET:%.*]] = icmp ne <3 x i8> [[X_HIGHBITS]], zeroinitializer ; CHECK-NEXT: ret <3 x i1> [[RET]] ; - %t0 = shl <3 x i8> , %y - %t1 = xor <3 x i8> %t0, + %t0 = shl <3 x i8> , %y + %t1 = xor <3 x i8> %t0, %t2 = and <3 x i8> %t1, %x %ret = icmp ne <3 x i8> %t2, %x ret <3 x i1> %ret diff --git a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v4-and-icmp-eq-to-icmp-ule.ll b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v4-and-icmp-eq-to-icmp-ule.ll index d13129c1248a..f48d284e085b 100644 --- a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v4-and-icmp-eq-to-icmp-ule.ll +++ b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v4-and-icmp-eq-to-icmp-ule.ll @@ -54,15 +54,15 @@ define <2 x i1> @p1_vec(<2 x i8> %x, <2 x i8> %y) { ret <2 x i1> %ret } -define <3 x i1> @p2_vec_undef0(<3 x i8> %x, <3 x i8> %y) { -; CHECK-LABEL: @p2_vec_undef0( -; CHECK-NEXT: [[T0:%.*]] = shl <3 x i8> , [[Y:%.*]] +define <3 x i1> @p2_vec_poison0(<3 x i8> %x, <3 x i8> %y) { +; CHECK-LABEL: @p2_vec_poison0( +; CHECK-NEXT: [[T0:%.*]] = shl nsw <3 x i8> , [[Y:%.*]] ; CHECK-NEXT: call void @use3i8(<3 x i8> [[T0]]) ; CHECK-NEXT: [[T1:%.*]] = lshr <3 x i8> , [[Y]] ; CHECK-NEXT: [[RET:%.*]] = icmp uge <3 x i8> [[T1]], [[X:%.*]] ; CHECK-NEXT: ret <3 x i1> [[RET]] ; - %t0 = shl <3 x i8> , %y + %t0 = shl <3 x i8> , %y call void @use3i8(<3 x i8> %t0) %t1 = lshr <3 x i8> %t0, %y %t2 = and <3 x i8> %t1, %x diff --git a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v4-and-icmp-ne-to-icmp-ugt.ll b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v4-and-icmp-ne-to-icmp-ugt.ll index a1517b36d0b9..f4b3c67164e4 100644 --- a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v4-and-icmp-ne-to-icmp-ugt.ll +++ b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v4-and-icmp-ne-to-icmp-ugt.ll @@ -54,15 +54,15 @@ define <2 x i1> @p1_vec(<2 x i8> %x, <2 x i8> %y) { ret <2 x i1> %ret } -define <3 x i1> @p2_vec_undef0(<3 x i8> %x, <3 x i8> %y) { -; CHECK-LABEL: @p2_vec_undef0( -; CHECK-NEXT: [[T0:%.*]] = shl <3 x i8> , [[Y:%.*]] +define <3 x i1> @p2_vec_poison0(<3 x i8> %x, <3 x i8> %y) { +; CHECK-LABEL: @p2_vec_poison0( +; CHECK-NEXT: [[T0:%.*]] = shl nsw <3 x i8> , [[Y:%.*]] ; CHECK-NEXT: call void @use3i8(<3 x i8> [[T0]]) ; CHECK-NEXT: [[T1:%.*]] = lshr <3 x i8> , [[Y]] ; CHECK-NEXT: [[RET:%.*]] = icmp ult <3 x i8> [[T1]], [[X:%.*]] ; CHECK-NEXT: ret <3 x i1> [[RET]] ; - %t0 = shl <3 x i8> , %y + %t0 = shl <3 x i8> , %y call void @use3i8(<3 x i8> %t0) %t1 = lshr <3 x i8> %t0, %y %t2 = and <3 x i8> %t1, %x diff --git a/llvm/test/Transforms/InstCombine/cast-int-icmp-eq-0.ll b/llvm/test/Transforms/InstCombine/cast-int-icmp-eq-0.ll index 9b51a7649992..7b6d07a14a30 100644 --- a/llvm/test/Transforms/InstCombine/cast-int-icmp-eq-0.ll +++ b/llvm/test/Transforms/InstCombine/cast-int-icmp-eq-0.ll @@ -603,7 +603,7 @@ define i1 @i16_cast_cmp_sgt_int_m1_sitofp_half(i16 %i) { ret i1 %cmp } -; Verify that vector types and vector constants including undef elements are transformed too. +; Verify that vector types and vector constants including poison elements are transformed too. define <3 x i1> @i32_cast_cmp_ne_int_0_sitofp_double_vec(<3 x i32> %i) { ; CHECK-LABEL: @i32_cast_cmp_ne_int_0_sitofp_double_vec( @@ -616,38 +616,38 @@ define <3 x i1> @i32_cast_cmp_ne_int_0_sitofp_double_vec(<3 x i32> %i) { ret <3 x i1> %cmp } -; TODO: Can we propagate the constant vector with undef element? +; TODO: Can we propagate the constant vector with poison element? -define <3 x i1> @i32_cast_cmp_eq_int_0_sitofp_float_vec_undef(<3 x i32> %i) { -; CHECK-LABEL: @i32_cast_cmp_eq_int_0_sitofp_float_vec_undef( +define <3 x i1> @i32_cast_cmp_eq_int_0_sitofp_float_vec_poison(<3 x i32> %i) { +; CHECK-LABEL: @i32_cast_cmp_eq_int_0_sitofp_float_vec_poison( ; CHECK-NEXT: [[CMP:%.*]] = icmp eq <3 x i32> [[I:%.*]], zeroinitializer ; CHECK-NEXT: ret <3 x i1> [[CMP]] ; %f = sitofp <3 x i32> %i to <3 x float> %b = bitcast <3 x float> %f to <3 x i32> - %cmp = icmp eq <3 x i32> %b, + %cmp = icmp eq <3 x i32> %b, ret <3 x i1> %cmp } -define <3 x i1> @i64_cast_cmp_slt_int_1_sitofp_half_vec_undef(<3 x i64> %i) { -; CHECK-LABEL: @i64_cast_cmp_slt_int_1_sitofp_half_vec_undef( +define <3 x i1> @i64_cast_cmp_slt_int_1_sitofp_half_vec_poison(<3 x i64> %i) { +; CHECK-LABEL: @i64_cast_cmp_slt_int_1_sitofp_half_vec_poison( ; CHECK-NEXT: [[CMP:%.*]] = icmp slt <3 x i64> [[I:%.*]], ; CHECK-NEXT: ret <3 x i1> [[CMP]] ; %f = sitofp <3 x i64> %i to <3 x half> %b = bitcast <3 x half> %f to <3 x i16> - %cmp = icmp slt <3 x i16> %b, + %cmp = icmp slt <3 x i16> %b, ret <3 x i1> %cmp } -define <3 x i1> @i16_cast_cmp_sgt_int_m1_sitofp_float_vec_undef(<3 x i16> %i) { -; CHECK-LABEL: @i16_cast_cmp_sgt_int_m1_sitofp_float_vec_undef( +define <3 x i1> @i16_cast_cmp_sgt_int_m1_sitofp_float_vec_poison(<3 x i16> %i) { +; CHECK-LABEL: @i16_cast_cmp_sgt_int_m1_sitofp_float_vec_poison( ; CHECK-NEXT: [[CMP:%.*]] = icmp sgt <3 x i16> [[I:%.*]], ; CHECK-NEXT: ret <3 x i1> [[CMP]] ; %f = sitofp <3 x i16> %i to <3 x float> %b = bitcast <3 x float> %f to <3 x i32> - %cmp = icmp sgt <3 x i32> %b, + %cmp = icmp sgt <3 x i32> %b, ret <3 x i1> %cmp } diff --git a/llvm/test/Transforms/InstCombine/cast-unsigned-icmp-eqcmp-0.ll b/llvm/test/Transforms/InstCombine/cast-unsigned-icmp-eqcmp-0.ll index 0752576fad45..1565fb7c0a6a 100644 --- a/llvm/test/Transforms/InstCombine/cast-unsigned-icmp-eqcmp-0.ll +++ b/llvm/test/Transforms/InstCombine/cast-unsigned-icmp-eqcmp-0.ll @@ -27,14 +27,14 @@ define <2 x i1> @i32_cast_cmp_eq_int_0_uitofp_float_vec(<2 x i32> %i) { ret <2 x i1> %cmp } -define <3 x i1> @i32_cast_cmp_eq_int_0_uitofp_float_vec_undef(<3 x i32> %i) { -; CHECK-LABEL: @i32_cast_cmp_eq_int_0_uitofp_float_vec_undef( +define <3 x i1> @i32_cast_cmp_eq_int_0_uitofp_float_vec_poison(<3 x i32> %i) { +; CHECK-LABEL: @i32_cast_cmp_eq_int_0_uitofp_float_vec_poison( ; CHECK-NEXT: [[CMP:%.*]] = icmp eq <3 x i32> [[I:%.*]], zeroinitializer ; CHECK-NEXT: ret <3 x i1> [[CMP]] ; %f = uitofp <3 x i32> %i to <3 x float> %b = bitcast <3 x float> %f to <3 x i32> - %cmp = icmp eq <3 x i32> %b, + %cmp = icmp eq <3 x i32> %b, ret <3 x i1> %cmp } @@ -60,14 +60,14 @@ define <2 x i1> @i32_cast_cmp_ne_int_0_uitofp_float_vec(<2 x i32> %i) { ret <2 x i1> %cmp } -define <3 x i1> @i32_cast_cmp_ne_int_0_uitofp_float_vec_undef(<3 x i32> %i) { -; CHECK-LABEL: @i32_cast_cmp_ne_int_0_uitofp_float_vec_undef( +define <3 x i1> @i32_cast_cmp_ne_int_0_uitofp_float_vec_poison(<3 x i32> %i) { +; CHECK-LABEL: @i32_cast_cmp_ne_int_0_uitofp_float_vec_poison( ; CHECK-NEXT: [[CMP:%.*]] = icmp ne <3 x i32> [[I:%.*]], zeroinitializer ; CHECK-NEXT: ret <3 x i1> [[CMP]] ; %f = uitofp <3 x i32> %i to <3 x float> %b = bitcast <3 x float> %f to <3 x i32> - %cmp = icmp ne <3 x i32> %b, + %cmp = icmp ne <3 x i32> %b, ret <3 x i1> %cmp } @@ -93,14 +93,14 @@ define <2 x i1> @i32_cast_cmp_eq_int_0_uitofp_double_vec(<2 x i32> %i) { ret <2 x i1> %cmp } -define <3 x i1> @i32_cast_cmp_eq_int_0_uitofp_double_vec_undef(<3 x i32> %i) { -; CHECK-LABEL: @i32_cast_cmp_eq_int_0_uitofp_double_vec_undef( +define <3 x i1> @i32_cast_cmp_eq_int_0_uitofp_double_vec_poison(<3 x i32> %i) { +; CHECK-LABEL: @i32_cast_cmp_eq_int_0_uitofp_double_vec_poison( ; CHECK-NEXT: [[CMP:%.*]] = icmp eq <3 x i32> [[I:%.*]], zeroinitializer ; CHECK-NEXT: ret <3 x i1> [[CMP]] ; %f = uitofp <3 x i32> %i to <3 x double> %b = bitcast <3 x double> %f to <3 x i64> - %cmp = icmp eq <3 x i64> %b, + %cmp = icmp eq <3 x i64> %b, ret <3 x i1> %cmp } @@ -126,14 +126,14 @@ define <2 x i1> @i32_cast_cmp_ne_int_0_uitofp_double_vec(<2 x i32> %i) { ret <2 x i1> %cmp } -define <3 x i1> @i32_cast_cmp_ne_int_0_uitofp_double_vec_undef(<3 x i32> %i) { -; CHECK-LABEL: @i32_cast_cmp_ne_int_0_uitofp_double_vec_undef( +define <3 x i1> @i32_cast_cmp_ne_int_0_uitofp_double_vec_poison(<3 x i32> %i) { +; CHECK-LABEL: @i32_cast_cmp_ne_int_0_uitofp_double_vec_poison( ; CHECK-NEXT: [[CMP:%.*]] = icmp ne <3 x i32> [[I:%.*]], zeroinitializer ; CHECK-NEXT: ret <3 x i1> [[CMP]] ; %f = uitofp <3 x i32> %i to <3 x double> %b = bitcast <3 x double> %f to <3 x i64> - %cmp = icmp ne <3 x i64> %b, + %cmp = icmp ne <3 x i64> %b, ret <3 x i1> %cmp } @@ -159,14 +159,14 @@ define <2 x i1> @i32_cast_cmp_eq_int_0_uitofp_half_vec(<2 x i32> %i) { ret <2 x i1> %cmp } -define <3 x i1> @i32_cast_cmp_eq_int_0_uitofp_half_vec_undef(<3 x i32> %i) { -; CHECK-LABEL: @i32_cast_cmp_eq_int_0_uitofp_half_vec_undef( +define <3 x i1> @i32_cast_cmp_eq_int_0_uitofp_half_vec_poison(<3 x i32> %i) { +; CHECK-LABEL: @i32_cast_cmp_eq_int_0_uitofp_half_vec_poison( ; CHECK-NEXT: [[CMP:%.*]] = icmp eq <3 x i32> [[I:%.*]], zeroinitializer ; CHECK-NEXT: ret <3 x i1> [[CMP]] ; %f = uitofp <3 x i32> %i to <3 x half> %b = bitcast <3 x half> %f to <3 x i16> - %cmp = icmp eq <3 x i16> %b, + %cmp = icmp eq <3 x i16> %b, ret <3 x i1> %cmp } @@ -192,13 +192,13 @@ define <2 x i1> @i32_cast_cmp_ne_int_0_uitofp_half_vec(<2 x i32> %i) { ret <2 x i1> %cmp } -define <3 x i1> @i32_cast_cmp_ne_int_0_uitofp_half_vec_undef(<3 x i32> %i) { -; CHECK-LABEL: @i32_cast_cmp_ne_int_0_uitofp_half_vec_undef( +define <3 x i1> @i32_cast_cmp_ne_int_0_uitofp_half_vec_poison(<3 x i32> %i) { +; CHECK-LABEL: @i32_cast_cmp_ne_int_0_uitofp_half_vec_poison( ; CHECK-NEXT: [[CMP:%.*]] = icmp ne <3 x i32> [[I:%.*]], zeroinitializer ; CHECK-NEXT: ret <3 x i1> [[CMP]] ; %f = uitofp <3 x i32> %i to <3 x half> %b = bitcast <3 x half> %f to <3 x i16> - %cmp = icmp ne <3 x i16> %b, + %cmp = icmp ne <3 x i16> %b, ret <3 x i1> %cmp } diff --git a/llvm/test/Transforms/InstCombine/cast.ll b/llvm/test/Transforms/InstCombine/cast.ll index d9c93ba27729..04a3e8931e62 100644 --- a/llvm/test/Transforms/InstCombine/cast.ll +++ b/llvm/test/Transforms/InstCombine/cast.ll @@ -508,18 +508,16 @@ define <2 x i16> @test40vec_nonuniform(<2 x i16> %a) { ret <2 x i16> %r } -define <2 x i16> @test40vec_undef(<2 x i16> %a) { -; ALL-LABEL: @test40vec_undef( -; ALL-NEXT: [[T:%.*]] = zext <2 x i16> [[A:%.*]] to <2 x i32> -; ALL-NEXT: [[T21:%.*]] = lshr <2 x i32> [[T]], -; ALL-NEXT: [[T5:%.*]] = shl <2 x i32> [[T]], -; ALL-NEXT: [[T32:%.*]] = or <2 x i32> [[T21]], [[T5]] -; ALL-NEXT: [[R:%.*]] = trunc <2 x i32> [[T32]] to <2 x i16> +define <2 x i16> @test40vec_poison(<2 x i16> %a) { +; ALL-LABEL: @test40vec_poison( +; ALL-NEXT: [[T21:%.*]] = lshr <2 x i16> [[A:%.*]], +; ALL-NEXT: [[T5:%.*]] = shl <2 x i16> [[A]], +; ALL-NEXT: [[R:%.*]] = or disjoint <2 x i16> [[T21]], [[T5]] ; ALL-NEXT: ret <2 x i16> [[R]] ; %t = zext <2 x i16> %a to <2 x i32> - %t21 = lshr <2 x i32> %t, - %t5 = shl <2 x i32> %t, + %t21 = lshr <2 x i32> %t, + %t5 = shl <2 x i32> %t, %t32 = or <2 x i32> %t21, %t5 %r = trunc <2 x i32> %t32 to <2 x i16> ret <2 x i16> %r @@ -1452,7 +1450,7 @@ define i32 @test89() { ; LE-LABEL: @test89( ; LE-NEXT: ret i32 6 ; - ret i32 bitcast (<2 x i16> to i32) + ret i32 bitcast (<2 x i16> to i32) } define <2 x i32> @test90() { @@ -1462,7 +1460,7 @@ define <2 x i32> @test90() { ; LE-LABEL: @test90( ; LE-NEXT: ret <2 x i32> ; - %t6 = bitcast <4 x half> to <2 x i32> + %t6 = bitcast <4 x half> to <2 x i32> ret <2 x i32> %t6 } @@ -1537,13 +1535,13 @@ define <2 x i8> @trunc_lshr_sext_uniform(<2 x i8> %A) { ret <2 x i8> %D } -define <2 x i8> @trunc_lshr_sext_uniform_undef(<2 x i8> %A) { -; ALL-LABEL: @trunc_lshr_sext_uniform_undef( -; ALL-NEXT: [[D:%.*]] = ashr <2 x i8> [[A:%.*]], +define <2 x i8> @trunc_lshr_sext_uniform_poison(<2 x i8> %A) { +; ALL-LABEL: @trunc_lshr_sext_uniform_poison( +; ALL-NEXT: [[D:%.*]] = ashr <2 x i8> [[A:%.*]], ; ALL-NEXT: ret <2 x i8> [[D]] ; %B = sext <2 x i8> %A to <2 x i32> - %C = lshr <2 x i32> %B, + %C = lshr <2 x i32> %B, %D = trunc <2 x i32> %C to <2 x i8> ret <2 x i8> %D } @@ -1559,13 +1557,13 @@ define <2 x i8> @trunc_lshr_sext_nonuniform(<2 x i8> %A) { ret <2 x i8> %D } -define <3 x i8> @trunc_lshr_sext_nonuniform_undef(<3 x i8> %A) { -; ALL-LABEL: @trunc_lshr_sext_nonuniform_undef( -; ALL-NEXT: [[D:%.*]] = ashr <3 x i8> [[A:%.*]], +define <3 x i8> @trunc_lshr_sext_nonuniform_poison(<3 x i8> %A) { +; ALL-LABEL: @trunc_lshr_sext_nonuniform_poison( +; ALL-NEXT: [[D:%.*]] = ashr <3 x i8> [[A:%.*]], ; ALL-NEXT: ret <3 x i8> [[D]] ; %B = sext <3 x i8> %A to <3 x i32> - %C = lshr <3 x i32> %B, + %C = lshr <3 x i32> %B, %D = trunc <3 x i32> %C to <3 x i8> ret <3 x i8> %D } @@ -2014,15 +2012,13 @@ define <2 x i8> @trunc_lshr_zext_uniform(<2 x i8> %A) { ret <2 x i8> %D } -define <2 x i8> @trunc_lshr_zext_uniform_undef(<2 x i8> %A) { -; ALL-LABEL: @trunc_lshr_zext_uniform_undef( -; ALL-NEXT: [[B:%.*]] = zext <2 x i8> [[A:%.*]] to <2 x i32> -; ALL-NEXT: [[C:%.*]] = lshr <2 x i32> [[B]], -; ALL-NEXT: [[D:%.*]] = trunc nuw <2 x i32> [[C]] to <2 x i8> +define <2 x i8> @trunc_lshr_zext_uniform_poison(<2 x i8> %A) { +; ALL-LABEL: @trunc_lshr_zext_uniform_poison( +; ALL-NEXT: [[D:%.*]] = lshr <2 x i8> [[A:%.*]], ; ALL-NEXT: ret <2 x i8> [[D]] ; %B = zext <2 x i8> %A to <2 x i32> - %C = lshr <2 x i32> %B, + %C = lshr <2 x i32> %B, %D = trunc <2 x i32> %C to <2 x i8> ret <2 x i8> %D } @@ -2038,15 +2034,13 @@ define <2 x i8> @trunc_lshr_zext_nonuniform(<2 x i8> %A) { ret <2 x i8> %D } -define <3 x i8> @trunc_lshr_zext_nonuniform_undef(<3 x i8> %A) { -; ALL-LABEL: @trunc_lshr_zext_nonuniform_undef( -; ALL-NEXT: [[B:%.*]] = zext <3 x i8> [[A:%.*]] to <3 x i32> -; ALL-NEXT: [[C:%.*]] = lshr <3 x i32> [[B]], -; ALL-NEXT: [[D:%.*]] = trunc nuw <3 x i32> [[C]] to <3 x i8> +define <3 x i8> @trunc_lshr_zext_nonuniform_poison(<3 x i8> %A) { +; ALL-LABEL: @trunc_lshr_zext_nonuniform_poison( +; ALL-NEXT: [[D:%.*]] = lshr <3 x i8> [[A:%.*]], ; ALL-NEXT: ret <3 x i8> [[D]] ; %B = zext <3 x i8> %A to <3 x i32> - %C = lshr <3 x i32> %B, + %C = lshr <3 x i32> %B, %D = trunc <3 x i32> %C to <3 x i8> ret <3 x i8> %D } diff --git a/llvm/test/Transforms/InstCombine/ctpop-cttz.ll b/llvm/test/Transforms/InstCombine/ctpop-cttz.ll index 5d27f374d892..70868554bdc1 100644 --- a/llvm/test/Transforms/InstCombine/ctpop-cttz.ll +++ b/llvm/test/Transforms/InstCombine/ctpop-cttz.ll @@ -116,14 +116,14 @@ define <2 x i32> @ctpop3v(<2 x i32> %0) { ret <2 x i32> %5 } -define <2 x i32> @ctpop3v_undef(<2 x i32> %0) { -; CHECK-LABEL: @ctpop3v_undef( +define <2 x i32> @ctpop3v_poison(<2 x i32> %0) { +; CHECK-LABEL: @ctpop3v_poison( ; CHECK-NEXT: [[TMP2:%.*]] = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> [[TMP0:%.*]], i1 false), !range [[RNG0]] ; CHECK-NEXT: ret <2 x i32> [[TMP2]] ; %2 = sub <2 x i32> zeroinitializer, %0 %3 = and <2 x i32> %2, %0 - %4 = add <2 x i32> %3, + %4 = add <2 x i32> %3, %5 = tail call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %4) ret <2 x i32> %5 } diff --git a/llvm/test/Transforms/InstCombine/ctpop.ll b/llvm/test/Transforms/InstCombine/ctpop.ll index 27194724b7d8..b3653e5071ba 100644 --- a/llvm/test/Transforms/InstCombine/ctpop.ll +++ b/llvm/test/Transforms/InstCombine/ctpop.ll @@ -155,28 +155,27 @@ define <2 x i32> @_parity_of_not_vec(<2 x i32> %x) { ret <2 x i32> %r } -define <2 x i32> @_parity_of_not_undef(<2 x i32> %x) { -; CHECK-LABEL: @_parity_of_not_undef( +define <2 x i32> @_parity_of_not_poison(<2 x i32> %x) { +; CHECK-LABEL: @_parity_of_not_poison( ; CHECK-NEXT: [[TMP1:%.*]] = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> [[X:%.*]]), !range [[RNG1]] ; CHECK-NEXT: [[R:%.*]] = and <2 x i32> [[TMP1]], ; CHECK-NEXT: ret <2 x i32> [[R]] ; - %neg = xor <2 x i32> %x, + %neg = xor <2 x i32> %x, %cnt = tail call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %neg) %r = and <2 x i32> %cnt, ret <2 x i32> %r } -define <2 x i32> @_parity_of_not_undef2(<2 x i32> %x) { -; CHECK-LABEL: @_parity_of_not_undef2( -; CHECK-NEXT: [[NEG:%.*]] = xor <2 x i32> [[X:%.*]], -; CHECK-NEXT: [[CNT:%.*]] = tail call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> [[NEG]]), !range [[RNG1]] -; CHECK-NEXT: [[R:%.*]] = and <2 x i32> [[CNT]], +define <2 x i32> @_parity_of_not_poison2(<2 x i32> %x) { +; CHECK-LABEL: @_parity_of_not_poison2( +; CHECK-NEXT: [[CNT:%.*]] = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> [[X:%.*]]), !range [[RNG1]] +; CHECK-NEXT: [[R:%.*]] = and <2 x i32> [[CNT]], ; CHECK-NEXT: ret <2 x i32> [[R]] ; %neg = xor <2 x i32> %x, %cnt = tail call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %neg) - %r = and <2 x i32> %cnt, + %r = and <2 x i32> %cnt, ret <2 x i32> %r } diff --git a/llvm/test/Transforms/InstCombine/fabs-as-int.ll b/llvm/test/Transforms/InstCombine/fabs-as-int.ll index f32c00e453f2..4e49ff159f87 100644 --- a/llvm/test/Transforms/InstCombine/fabs-as-int.ll +++ b/llvm/test/Transforms/InstCombine/fabs-as-int.ll @@ -137,15 +137,15 @@ define <2 x i32> @not_fabs_as_int_v2f32_nonsplat(<2 x float> %x) { ret <2 x i32> %and } -define <3 x i32> @fabs_as_int_v3f32_undef(<3 x float> %x) { -; CHECK-LABEL: define <3 x i32> @fabs_as_int_v3f32_undef +define <3 x i32> @fabs_as_int_v3f32_poison(<3 x float> %x) { +; CHECK-LABEL: define <3 x i32> @fabs_as_int_v3f32_poison ; CHECK-SAME: (<3 x float> [[X:%.*]]) { ; CHECK-NEXT: [[TMP1:%.*]] = call <3 x float> @llvm.fabs.v3f32(<3 x float> [[X]]) ; CHECK-NEXT: [[AND:%.*]] = bitcast <3 x float> [[TMP1]] to <3 x i32> ; CHECK-NEXT: ret <3 x i32> [[AND]] ; %bc = bitcast <3 x float> %x to <3 x i32> - %and = and <3 x i32> %bc, + %and = and <3 x i32> %bc, ret <3 x i32> %and } diff --git a/llvm/test/Transforms/InstCombine/fabs.ll b/llvm/test/Transforms/InstCombine/fabs.ll index 7e380c2e4590..5ec65784e7a3 100644 --- a/llvm/test/Transforms/InstCombine/fabs.ll +++ b/llvm/test/Transforms/InstCombine/fabs.ll @@ -321,7 +321,7 @@ define <2 x float> @select_fcmp_nnan_ole_negzero(<2 x float> %x) { ; CHECK-NEXT: ret <2 x float> [[FABS]] ; %lezero = fcmp ole <2 x float> %x, - %negx = fsub nnan <2 x float> , %x + %negx = fsub nnan <2 x float> , %x %fabs = select <2 x i1> %lezero, <2 x float> %negx, <2 x float> %x ret <2 x float> %fabs } @@ -332,7 +332,7 @@ define <2 x float> @select_nnan_fcmp_nnan_ole_negzero(<2 x float> %x) { ; CHECK-NEXT: ret <2 x float> [[FABS]] ; %lezero = fcmp ole <2 x float> %x, - %negx = fsub nnan <2 x float> , %x + %negx = fsub nnan <2 x float> , %x %fabs = select nnan <2 x i1> %lezero, <2 x float> %negx, <2 x float> %x ret <2 x float> %fabs } diff --git a/llvm/test/Transforms/InstCombine/fast-math.ll b/llvm/test/Transforms/InstCombine/fast-math.ll index 916955e34efa..83f2091244e5 100644 --- a/llvm/test/Transforms/InstCombine/fast-math.ll +++ b/llvm/test/Transforms/InstCombine/fast-math.ll @@ -541,12 +541,12 @@ define float @fneg2(float %x) { ret float %sub } -define <2 x float> @fneg2_vec_undef(<2 x float> %x) { -; CHECK-LABEL: @fneg2_vec_undef( +define <2 x float> @fneg2_vec_poison(<2 x float> %x) { +; CHECK-LABEL: @fneg2_vec_poison( ; CHECK-NEXT: [[SUB:%.*]] = fneg nsz <2 x float> [[X:%.*]] ; CHECK-NEXT: ret <2 x float> [[SUB]] ; - %sub = fsub nsz <2 x float> , %x + %sub = fsub nsz <2 x float> , %x ret <2 x float> %sub } diff --git a/llvm/test/Transforms/InstCombine/fcmp-special.ll b/llvm/test/Transforms/InstCombine/fcmp-special.ll index 88bfe930ffdd..64bc86f4266c 100644 --- a/llvm/test/Transforms/InstCombine/fcmp-special.ll +++ b/llvm/test/Transforms/InstCombine/fcmp-special.ll @@ -144,21 +144,21 @@ define <2 x i1> @uno_vec_with_nan(<2 x double> %x) { ret <2 x i1> %f } -define <2 x i1> @uno_vec_with_undef(<2 x double> %x) { -; CHECK-LABEL: @uno_vec_with_undef( +define <2 x i1> @uno_vec_with_poison(<2 x double> %x) { +; CHECK-LABEL: @uno_vec_with_poison( ; CHECK-NEXT: [[F:%.*]] = fcmp uno <2 x double> [[X:%.*]], zeroinitializer ; CHECK-NEXT: ret <2 x i1> [[F]] ; - %f = fcmp uno <2 x double> %x, + %f = fcmp uno <2 x double> %x, ret <2 x i1> %f } -define <2 x i1> @ord_vec_with_undef(<2 x double> %x) { -; CHECK-LABEL: @ord_vec_with_undef( -; CHECK-NEXT: [[F:%.*]] = fcmp ord <2 x double> [[X:%.*]], +define <2 x i1> @ord_vec_with_poison(<2 x double> %x) { +; CHECK-LABEL: @ord_vec_with_poison( +; CHECK-NEXT: [[F:%.*]] = fcmp ord <2 x double> [[X:%.*]], ; CHECK-NEXT: ret <2 x i1> [[F]] ; - %f = fcmp ord <2 x double> %x, + %f = fcmp ord <2 x double> %x, ret <2 x i1> %f } @@ -224,12 +224,12 @@ define <2 x i1> @negative_zero_olt_vec(<2 x float> %x) { ret <2 x i1> %r } -define <2 x i1> @negative_zero_une_vec_undef(<2 x double> %x) { -; CHECK-LABEL: @negative_zero_une_vec_undef( +define <2 x i1> @negative_zero_une_vec_poison(<2 x double> %x) { +; CHECK-LABEL: @negative_zero_une_vec_poison( ; CHECK-NEXT: [[R:%.*]] = fcmp nnan une <2 x double> [[X:%.*]], zeroinitializer ; CHECK-NEXT: ret <2 x i1> [[R]] ; - %r = fcmp nnan une <2 x double> %x, + %r = fcmp nnan une <2 x double> %x, ret <2 x i1> %r } diff --git a/llvm/test/Transforms/InstCombine/fcmp.ll b/llvm/test/Transforms/InstCombine/fcmp.ll index 069512b0f2d8..389264e2f707 100644 --- a/llvm/test/Transforms/InstCombine/fcmp.ll +++ b/llvm/test/Transforms/InstCombine/fcmp.ll @@ -102,12 +102,12 @@ define <2 x i1> @unary_fneg_constant_swap_pred_vec(<2 x float> %x) { ret <2 x i1> %cmp } -define <2 x i1> @fneg_constant_swap_pred_vec_undef(<2 x float> %x) { -; CHECK-LABEL: @fneg_constant_swap_pred_vec_undef( +define <2 x i1> @fneg_constant_swap_pred_vec_poison(<2 x float> %x) { +; CHECK-LABEL: @fneg_constant_swap_pred_vec_poison( ; CHECK-NEXT: [[CMP:%.*]] = fcmp olt <2 x float> [[X:%.*]], ; CHECK-NEXT: ret <2 x i1> [[CMP]] ; - %neg = fsub <2 x float> , %x + %neg = fsub <2 x float> , %x %cmp = fcmp ogt <2 x float> %neg, ret <2 x i1> %cmp } @@ -234,34 +234,34 @@ define <2 x i1> @fneg_unary_fneg_swap_pred_vec(<2 x float> %x, <2 x float> %y) { ret <2 x i1> %cmp } -define <2 x i1> @fneg_fneg_swap_pred_vec_undef(<2 x float> %x, <2 x float> %y) { -; CHECK-LABEL: @fneg_fneg_swap_pred_vec_undef( +define <2 x i1> @fneg_fneg_swap_pred_vec_poison(<2 x float> %x, <2 x float> %y) { +; CHECK-LABEL: @fneg_fneg_swap_pred_vec_poison( ; CHECK-NEXT: [[CMP:%.*]] = fcmp ogt <2 x float> [[X:%.*]], [[Y:%.*]] ; CHECK-NEXT: ret <2 x i1> [[CMP]] ; - %neg1 = fsub <2 x float> , %x - %neg2 = fsub <2 x float> , %y + %neg1 = fsub <2 x float> , %x + %neg2 = fsub <2 x float> , %y %cmp = fcmp olt <2 x float> %neg1, %neg2 ret <2 x i1> %cmp } -define <2 x i1> @unary_fneg_fneg_swap_pred_vec_undef(<2 x float> %x, <2 x float> %y) { -; CHECK-LABEL: @unary_fneg_fneg_swap_pred_vec_undef( +define <2 x i1> @unary_fneg_fneg_swap_pred_vec_poison(<2 x float> %x, <2 x float> %y) { +; CHECK-LABEL: @unary_fneg_fneg_swap_pred_vec_poison( ; CHECK-NEXT: [[CMP:%.*]] = fcmp ogt <2 x float> [[X:%.*]], [[Y:%.*]] ; CHECK-NEXT: ret <2 x i1> [[CMP]] ; %neg1 = fneg <2 x float> %x - %neg2 = fsub <2 x float> , %y + %neg2 = fsub <2 x float> , %y %cmp = fcmp olt <2 x float> %neg1, %neg2 ret <2 x i1> %cmp } -define <2 x i1> @fneg_unary_fneg_swap_pred_vec_undef(<2 x float> %x, <2 x float> %y) { -; CHECK-LABEL: @fneg_unary_fneg_swap_pred_vec_undef( +define <2 x i1> @fneg_unary_fneg_swap_pred_vec_poison(<2 x float> %x, <2 x float> %y) { +; CHECK-LABEL: @fneg_unary_fneg_swap_pred_vec_poison( ; CHECK-NEXT: [[CMP:%.*]] = fcmp ogt <2 x float> [[X:%.*]], [[Y:%.*]] ; CHECK-NEXT: ret <2 x i1> [[CMP]] ; - %neg1 = fsub <2 x float> , %x + %neg1 = fsub <2 x float> , %x %neg2 = fneg <2 x float> %y %cmp = fcmp olt <2 x float> %neg1, %neg2 ret <2 x i1> %cmp diff --git a/llvm/test/Transforms/InstCombine/fdiv.ll b/llvm/test/Transforms/InstCombine/fdiv.ll index a0710c2bb048..ca11685c9841 100644 --- a/llvm/test/Transforms/InstCombine/fdiv.ll +++ b/llvm/test/Transforms/InstCombine/fdiv.ll @@ -141,12 +141,12 @@ define <2 x float> @not_exact_inverse_vec_arcp(<2 x float> %x) { ret <2 x float> %div } -define <2 x float> @not_exact_inverse_vec_arcp_with_undef_elt(<2 x float> %x) { -; CHECK-LABEL: @not_exact_inverse_vec_arcp_with_undef_elt( -; CHECK-NEXT: [[DIV:%.*]] = fdiv arcp <2 x float> [[X:%.*]], +define <2 x float> @not_exact_inverse_vec_arcp_with_poison_elt(<2 x float> %x) { +; CHECK-LABEL: @not_exact_inverse_vec_arcp_with_poison_elt( +; CHECK-NEXT: [[DIV:%.*]] = fdiv arcp <2 x float> [[X:%.*]], ; CHECK-NEXT: ret <2 x float> [[DIV]] ; - %div = fdiv arcp <2 x float> %x, + %div = fdiv arcp <2 x float> %x, ret <2 x float> %div } @@ -333,13 +333,13 @@ define <2 x float> @unary_fneg_fneg_vec(<2 x float> %x, <2 x float> %y) { ret <2 x float> %div } -define <2 x float> @fneg_fneg_vec_undef_elts(<2 x float> %x, <2 x float> %y) { -; CHECK-LABEL: @fneg_fneg_vec_undef_elts( +define <2 x float> @fneg_fneg_vec_poison_elts(<2 x float> %x, <2 x float> %y) { +; CHECK-LABEL: @fneg_fneg_vec_poison_elts( ; CHECK-NEXT: [[DIV:%.*]] = fdiv <2 x float> [[X:%.*]], [[Y:%.*]] ; CHECK-NEXT: ret <2 x float> [[DIV]] ; - %xneg = fsub <2 x float> , %x - %yneg = fsub <2 x float> , %y + %xneg = fsub <2 x float> , %x + %yneg = fsub <2 x float> , %y %div = fdiv <2 x float> %xneg, %yneg ret <2 x float> %div } @@ -404,12 +404,12 @@ define <2 x float> @unary_fneg_dividend_constant_divisor_vec(<2 x float> %x) { ret <2 x float> %div } -define <2 x float> @fneg_dividend_constant_divisor_vec_undef_elt(<2 x float> %x) { -; CHECK-LABEL: @fneg_dividend_constant_divisor_vec_undef_elt( +define <2 x float> @fneg_dividend_constant_divisor_vec_poison_elt(<2 x float> %x) { +; CHECK-LABEL: @fneg_dividend_constant_divisor_vec_poison_elt( ; CHECK-NEXT: [[DIV:%.*]] = fdiv ninf <2 x float> [[X:%.*]], ; CHECK-NEXT: ret <2 x float> [[DIV]] ; - %neg = fsub <2 x float> , %x + %neg = fsub <2 x float> , %x %div = fdiv ninf <2 x float> %neg, ret <2 x float> %div } diff --git a/llvm/test/Transforms/InstCombine/fma.ll b/llvm/test/Transforms/InstCombine/fma.ll index 8b413ae6f664..cf3d7f3c525a 100644 --- a/llvm/test/Transforms/InstCombine/fma.ll +++ b/llvm/test/Transforms/InstCombine/fma.ll @@ -60,13 +60,13 @@ define <2 x float> @fma_unary_fneg_x_unary_fneg_y_vec(<2 x float> %x, <2 x float ret <2 x float> %fma } -define <2 x float> @fma_fneg_x_fneg_y_vec_undef(<2 x float> %x, <2 x float> %y, <2 x float> %z) { -; CHECK-LABEL: @fma_fneg_x_fneg_y_vec_undef( +define <2 x float> @fma_fneg_x_fneg_y_vec_poison(<2 x float> %x, <2 x float> %y, <2 x float> %z) { +; CHECK-LABEL: @fma_fneg_x_fneg_y_vec_poison( ; CHECK-NEXT: [[FMA:%.*]] = call <2 x float> @llvm.fma.v2f32(<2 x float> [[X:%.*]], <2 x float> [[Y:%.*]], <2 x float> [[Z:%.*]]) ; CHECK-NEXT: ret <2 x float> [[FMA]] ; - %xn = fsub <2 x float> , %x - %yn = fsub <2 x float> , %y + %xn = fsub <2 x float> , %x + %yn = fsub <2 x float> , %y %fma = call <2 x float> @llvm.fma.v2f32(<2 x float> %xn, <2 x float> %yn, <2 x float> %z) ret <2 x float> %fma } diff --git a/llvm/test/Transforms/InstCombine/fmul.ll b/llvm/test/Transforms/InstCombine/fmul.ll index 39f9e74f899d..e9c86a127049 100644 --- a/llvm/test/Transforms/InstCombine/fmul.ll +++ b/llvm/test/Transforms/InstCombine/fmul.ll @@ -42,12 +42,12 @@ define <2 x float> @unary_neg_constant_vec(<2 x float> %x) { ret <2 x float> %mul } -define <2 x float> @neg_constant_vec_undef(<2 x float> %x) { -; CHECK-LABEL: @neg_constant_vec_undef( +define <2 x float> @neg_constant_vec_poison(<2 x float> %x) { +; CHECK-LABEL: @neg_constant_vec_poison( ; CHECK-NEXT: [[MUL:%.*]] = fmul ninf <2 x float> [[X:%.*]], ; CHECK-NEXT: ret <2 x float> [[MUL]] ; - %sub = fsub <2 x float> , %x + %sub = fsub <2 x float> , %x %mul = fmul ninf <2 x float> %sub, ret <2 x float> %mul } @@ -162,34 +162,34 @@ define <2 x float> @neg_unary_neg_vec(<2 x float> %x, <2 x float> %y) { ret <2 x float> %mul } -define <2 x float> @neg_neg_vec_undef(<2 x float> %x, <2 x float> %y) { -; CHECK-LABEL: @neg_neg_vec_undef( +define <2 x float> @neg_neg_vec_poison(<2 x float> %x, <2 x float> %y) { +; CHECK-LABEL: @neg_neg_vec_poison( ; CHECK-NEXT: [[MUL:%.*]] = fmul arcp <2 x float> [[X:%.*]], [[Y:%.*]] ; CHECK-NEXT: ret <2 x float> [[MUL]] ; - %sub1 = fsub <2 x float> , %x - %sub2 = fsub <2 x float> , %y + %sub1 = fsub <2 x float> , %x + %sub2 = fsub <2 x float> , %y %mul = fmul arcp <2 x float> %sub1, %sub2 ret <2 x float> %mul } -define <2 x float> @unary_neg_neg_vec_undef(<2 x float> %x, <2 x float> %y) { -; CHECK-LABEL: @unary_neg_neg_vec_undef( +define <2 x float> @unary_neg_neg_vec_poison(<2 x float> %x, <2 x float> %y) { +; CHECK-LABEL: @unary_neg_neg_vec_poison( ; CHECK-NEXT: [[MUL:%.*]] = fmul arcp <2 x float> [[X:%.*]], [[Y:%.*]] ; CHECK-NEXT: ret <2 x float> [[MUL]] ; %neg = fneg <2 x float> %x - %sub = fsub <2 x float> , %y + %sub = fsub <2 x float> , %y %mul = fmul arcp <2 x float> %neg, %sub ret <2 x float> %mul } -define <2 x float> @neg_unary_neg_vec_undef(<2 x float> %x, <2 x float> %y) { -; CHECK-LABEL: @neg_unary_neg_vec_undef( +define <2 x float> @neg_unary_neg_vec_poison(<2 x float> %x, <2 x float> %y) { +; CHECK-LABEL: @neg_unary_neg_vec_poison( ; CHECK-NEXT: [[MUL:%.*]] = fmul arcp <2 x float> [[X:%.*]], [[Y:%.*]] ; CHECK-NEXT: ret <2 x float> [[MUL]] ; - %sub = fsub <2 x float> , %x + %sub = fsub <2 x float> , %x %neg = fneg <2 x float> %y %mul = fmul arcp <2 x float> %sub, %neg ret <2 x float> %mul @@ -322,13 +322,13 @@ define <2 x float> @unary_neg_mul_vec(<2 x float> %x, <2 x float> %y) { ret <2 x float> %mul } -define <2 x float> @neg_mul_vec_undef(<2 x float> %x, <2 x float> %y) { -; CHECK-LABEL: @neg_mul_vec_undef( +define <2 x float> @neg_mul_vec_poison(<2 x float> %x, <2 x float> %y) { +; CHECK-LABEL: @neg_mul_vec_poison( ; CHECK-NEXT: [[SUB:%.*]] = fneg <2 x float> [[X:%.*]] ; CHECK-NEXT: [[MUL:%.*]] = fmul <2 x float> [[SUB]], [[Y:%.*]] ; CHECK-NEXT: ret <2 x float> [[MUL]] ; - %sub = fsub <2 x float> , %x + %sub = fsub <2 x float> , %x %mul = fmul <2 x float> %sub, %y ret <2 x float> %mul } @@ -388,9 +388,9 @@ define void @test8(ptr %inout, i1 %c1) { entry: %0 = load i32, ptr %inout, align 4 %conv = uitofp i32 %0 to float - %vecinit = insertelement <4 x float> , float %conv, i32 3 + %vecinit = insertelement <4 x float> , float %conv, i32 3 %sub = fsub <4 x float> , %vecinit - %1 = shufflevector <4 x float> %sub, <4 x float> undef, <4 x i32> + %1 = shufflevector <4 x float> %sub, <4 x float> poison, <4 x i32> %mul = fmul <4 x float> zeroinitializer, %1 br label %for.cond @@ -742,7 +742,7 @@ define <4 x float> @fdiv_constant_denominator_fmul_vec_constexpr(<4 x float> %x) ; CHECK-NEXT: [[T3:%.*]] = fmul reassoc <4 x float> [[X:%.*]], ; CHECK-NEXT: ret <4 x float> [[T3]] ; - %constExprMul = bitcast i128 trunc (i160 bitcast (<5 x float> to i160) to i128) to <4 x float> + %constExprMul = bitcast i128 trunc (i160 bitcast (<5 x float> to i160) to i128) to <4 x float> %t1 = fdiv reassoc <4 x float> %x, %t3 = fmul reassoc <4 x float> %t1, %constExprMul ret <4 x float> %t3 @@ -1270,7 +1270,7 @@ define @mul_scalable_splat_zero( %z) { ; CHECK-LABEL: @mul_scalable_splat_zero( ; CHECK-NEXT: ret zeroinitializer ; - %shuf = shufflevector insertelement ( undef, float 0.0, i32 0), undef, zeroinitializer + %shuf = shufflevector insertelement ( poison, float 0.0, i32 0), poison, zeroinitializer %t3 = fmul fast %shuf, %z ret %t3 } @@ -1393,7 +1393,7 @@ define <3 x float> @mul_neg_zero_nnan_ninf_vec(<3 x float> nofpclass(inf nan) %a ; CHECK-NEXT: ret <3 x float> [[RET]] ; entry: - %ret = fmul <3 x float> %a, + %ret = fmul <3 x float> %a, ret <3 x float> %ret } diff --git a/llvm/test/Transforms/InstCombine/fneg-as-int.ll b/llvm/test/Transforms/InstCombine/fneg-as-int.ll index d28e599cacf3..e3067b0d0246 100644 --- a/llvm/test/Transforms/InstCombine/fneg-as-int.ll +++ b/llvm/test/Transforms/InstCombine/fneg-as-int.ll @@ -139,15 +139,15 @@ define <2 x i32> @not_fneg_as_int_v2f32_nonsplat(<2 x float> %x) { ret <2 x i32> %xor } -define <3 x i32> @fneg_as_int_v3f32_undef(<3 x float> %x) { -; CHECK-LABEL: define <3 x i32> @fneg_as_int_v3f32_undef +define <3 x i32> @fneg_as_int_v3f32_poison(<3 x float> %x) { +; CHECK-LABEL: define <3 x i32> @fneg_as_int_v3f32_poison ; CHECK-SAME: (<3 x float> [[X:%.*]]) { ; CHECK-NEXT: [[TMP1:%.*]] = fneg <3 x float> [[X]] ; CHECK-NEXT: [[XOR:%.*]] = bitcast <3 x float> [[TMP1]] to <3 x i32> ; CHECK-NEXT: ret <3 x i32> [[XOR]] ; %bc = bitcast <3 x float> %x to <3 x i32> - %xor = xor <3 x i32> %bc, + %xor = xor <3 x i32> %bc, ret <3 x i32> %xor } diff --git a/llvm/test/Transforms/InstCombine/fneg-fabs-as-int.ll b/llvm/test/Transforms/InstCombine/fneg-fabs-as-int.ll index 9aa8d4944e39..8c3e6958fe08 100644 --- a/llvm/test/Transforms/InstCombine/fneg-fabs-as-int.ll +++ b/llvm/test/Transforms/InstCombine/fneg-fabs-as-int.ll @@ -158,8 +158,8 @@ define <2 x i32> @not_fneg_fabs_as_int_v2f32_nonsplat(<2 x float> %x) { ret <2 x i32> %or } -define <3 x i32> @fneg_fabs_as_int_v3f32_undef(<3 x float> %x) { -; CHECK-LABEL: define <3 x i32> @fneg_fabs_as_int_v3f32_undef +define <3 x i32> @fneg_fabs_as_int_v3f32_poison(<3 x float> %x) { +; CHECK-LABEL: define <3 x i32> @fneg_fabs_as_int_v3f32_poison ; CHECK-SAME: (<3 x float> [[X:%.*]]) { ; CHECK-NEXT: [[TMP1:%.*]] = call <3 x float> @llvm.fabs.v3f32(<3 x float> [[X]]) ; CHECK-NEXT: [[TMP2:%.*]] = fneg <3 x float> [[TMP1]] @@ -167,7 +167,7 @@ define <3 x i32> @fneg_fabs_as_int_v3f32_undef(<3 x float> %x) { ; CHECK-NEXT: ret <3 x i32> [[OR]] ; %bc = bitcast <3 x float> %x to <3 x i32> - %or = or <3 x i32> %bc, + %or = or <3 x i32> %bc, ret <3 x i32> %or } diff --git a/llvm/test/Transforms/InstCombine/fneg.ll b/llvm/test/Transforms/InstCombine/fneg.ll index ed68ba50d36e..7c9289c44711 100644 --- a/llvm/test/Transforms/InstCombine/fneg.ll +++ b/llvm/test/Transforms/InstCombine/fneg.ll @@ -87,24 +87,24 @@ define float @fmul_fneg_extra_use(float %x) { ret float %r } -; Try a vector. Use special constants (NaN, INF, undef) because they don't change anything. +; Try a vector. Use special constants (NaN, INF, poison) because they don't change anything. define <4 x double> @fmul_fsub_vec(<4 x double> %x) { ; CHECK-LABEL: @fmul_fsub_vec( -; CHECK-NEXT: [[R:%.*]] = fmul <4 x double> [[X:%.*]], +; CHECK-NEXT: [[R:%.*]] = fmul <4 x double> [[X:%.*]], ; CHECK-NEXT: ret <4 x double> [[R]] ; - %m = fmul <4 x double> %x, + %m = fmul <4 x double> %x, %r = fsub <4 x double> , %m ret <4 x double> %r } define <4 x double> @fmul_fneg_vec(<4 x double> %x) { ; CHECK-LABEL: @fmul_fneg_vec( -; CHECK-NEXT: [[R:%.*]] = fmul <4 x double> [[X:%.*]], +; CHECK-NEXT: [[R:%.*]] = fmul <4 x double> [[X:%.*]], ; CHECK-NEXT: ret <4 x double> [[R]] ; - %m = fmul <4 x double> %x, + %m = fmul <4 x double> %x, %r = fneg <4 x double> %m ret <4 x double> %r } @@ -181,24 +181,24 @@ define float @fdiv_op1_constant_fneg_extra_use(float %x) { ret float %r } -; Try a vector. Use special constants (NaN, INF, undef) because they don't change anything. +; Try a vector. Use special constants (NaN, INF, poison) because they don't change anything. define <4 x double> @fdiv_op1_constant_fsub_vec(<4 x double> %x) { ; CHECK-LABEL: @fdiv_op1_constant_fsub_vec( -; CHECK-NEXT: [[R:%.*]] = fdiv <4 x double> [[X:%.*]], +; CHECK-NEXT: [[R:%.*]] = fdiv <4 x double> [[X:%.*]], ; CHECK-NEXT: ret <4 x double> [[R]] ; - %d = fdiv <4 x double> %x, + %d = fdiv <4 x double> %x, %r = fsub <4 x double> , %d ret <4 x double> %r } define <4 x double> @fdiv_op1_constant_fneg_vec(<4 x double> %x) { ; CHECK-LABEL: @fdiv_op1_constant_fneg_vec( -; CHECK-NEXT: [[R:%.*]] = fdiv <4 x double> [[X:%.*]], +; CHECK-NEXT: [[R:%.*]] = fdiv <4 x double> [[X:%.*]], ; CHECK-NEXT: ret <4 x double> [[R]] ; - %d = fdiv <4 x double> %x, + %d = fdiv <4 x double> %x, %r = fneg <4 x double> %d ret <4 x double> %r } @@ -335,24 +335,24 @@ define float @fdiv_op0_constant_fneg_extra_use(float %x) { ret float %r } -; Try a vector. Use special constants (NaN, INF, undef) because they don't change anything. +; Try a vector. Use special constants (NaN, INF, poison) because they don't change anything. define <4 x double> @fdiv_op0_constant_fsub_vec(<4 x double> %x) { ; CHECK-LABEL: @fdiv_op0_constant_fsub_vec( -; CHECK-NEXT: [[R:%.*]] = fdiv <4 x double> , [[X:%.*]] +; CHECK-NEXT: [[R:%.*]] = fdiv <4 x double> , [[X:%.*]] ; CHECK-NEXT: ret <4 x double> [[R]] ; - %d = fdiv <4 x double> , %x + %d = fdiv <4 x double> , %x %r = fsub <4 x double> , %d ret <4 x double> %r } define <4 x double> @fdiv_op0_constant_fneg_vec(<4 x double> %x) { ; CHECK-LABEL: @fdiv_op0_constant_fneg_vec( -; CHECK-NEXT: [[R:%.*]] = fdiv <4 x double> , [[X:%.*]] +; CHECK-NEXT: [[R:%.*]] = fdiv <4 x double> , [[X:%.*]] ; CHECK-NEXT: ret <4 x double> [[R]] ; - %d = fdiv <4 x double> , %x + %d = fdiv <4 x double> , %x %r = fneg <4 x double> %d ret <4 x double> %r } @@ -584,11 +584,11 @@ define <2 x float> @fneg_nsz_fadd_constant_vec(<2 x float> %x) { define <2 x float> @fake_fneg_nsz_fadd_constant_vec(<2 x float> %x) { ; CHECK-LABEL: @fake_fneg_nsz_fadd_constant_vec( -; CHECK-NEXT: [[R:%.*]] = fsub nsz <2 x float> , [[X:%.*]] +; CHECK-NEXT: [[R:%.*]] = fsub nsz <2 x float> , [[X:%.*]] ; CHECK-NEXT: ret <2 x float> [[R]] ; - %a = fadd <2 x float> %x, - %r = fsub nsz <2 x float> , %a + %a = fadd <2 x float> %x, + %r = fsub nsz <2 x float> , %a ret <2 x float> %r } diff --git a/llvm/test/Transforms/InstCombine/fold-inc-of-add-of-not-x-and-y-to-sub-x-from-y.ll b/llvm/test/Transforms/InstCombine/fold-inc-of-add-of-not-x-and-y-to-sub-x-from-y.ll index b482cfdfde19..1fd570bf2635 100644 --- a/llvm/test/Transforms/InstCombine/fold-inc-of-add-of-not-x-and-y-to-sub-x-from-y.ll +++ b/llvm/test/Transforms/InstCombine/fold-inc-of-add-of-not-x-and-y-to-sub-x-from-y.ll @@ -36,36 +36,36 @@ define <4 x i32> @t1_vec_splat(<4 x i32> %x, <4 x i32> %y) { ret <4 x i32> %t2 } -define <4 x i32> @t2_vec_undef0(<4 x i32> %x, <4 x i32> %y) { -; CHECK-LABEL: @t2_vec_undef0( +define <4 x i32> @t2_vec_poison0(<4 x i32> %x, <4 x i32> %y) { +; CHECK-LABEL: @t2_vec_poison0( ; CHECK-NEXT: [[T2:%.*]] = sub <4 x i32> [[Y:%.*]], [[X:%.*]] ; CHECK-NEXT: ret <4 x i32> [[T2]] ; - %t0 = xor <4 x i32> %x, + %t0 = xor <4 x i32> %x, %t1 = add <4 x i32> %t0, %y %t2 = add <4 x i32> %t1, ret <4 x i32> %t2 } -define <4 x i32> @t3_vec_undef1(<4 x i32> %x, <4 x i32> %y) { -; CHECK-LABEL: @t3_vec_undef1( +define <4 x i32> @t3_vec_poison1(<4 x i32> %x, <4 x i32> %y) { +; CHECK-LABEL: @t3_vec_poison1( ; CHECK-NEXT: [[T2:%.*]] = sub <4 x i32> [[Y:%.*]], [[X:%.*]] ; CHECK-NEXT: ret <4 x i32> [[T2]] ; %t0 = xor <4 x i32> %x, %t1 = add <4 x i32> %t0, %y - %t2 = add <4 x i32> %t1, + %t2 = add <4 x i32> %t1, ret <4 x i32> %t2 } -define <4 x i32> @t4_vec_undef2(<4 x i32> %x, <4 x i32> %y) { -; CHECK-LABEL: @t4_vec_undef2( +define <4 x i32> @t4_vec_poison2(<4 x i32> %x, <4 x i32> %y) { +; CHECK-LABEL: @t4_vec_poison2( ; CHECK-NEXT: [[T2:%.*]] = sub <4 x i32> [[Y:%.*]], [[X:%.*]] ; CHECK-NEXT: ret <4 x i32> [[T2]] ; - %t0 = xor <4 x i32> %x, + %t0 = xor <4 x i32> %x, %t1 = add <4 x i32> %t0, %y - %t2 = add <4 x i32> %t1, + %t2 = add <4 x i32> %t1, ret <4 x i32> %t2 } diff --git a/llvm/test/Transforms/InstCombine/fold-sub-of-not-to-inc-of-add.ll b/llvm/test/Transforms/InstCombine/fold-sub-of-not-to-inc-of-add.ll index 6f311f05fb01..af580ba57513 100644 --- a/llvm/test/Transforms/InstCombine/fold-sub-of-not-to-inc-of-add.ll +++ b/llvm/test/Transforms/InstCombine/fold-sub-of-not-to-inc-of-add.ll @@ -50,13 +50,13 @@ define <4 x i32> @p1_vector_splat(<4 x i32> %x, <4 x i32> %y) { ret <4 x i32> %t1 } -define <4 x i32> @p2_vector_undef(<4 x i32> %x, <4 x i32> %y) { -; CHECK-LABEL: @p2_vector_undef( +define <4 x i32> @p2_vector_poison(<4 x i32> %x, <4 x i32> %y) { +; CHECK-LABEL: @p2_vector_poison( ; CHECK-NEXT: [[T0_NEG:%.*]] = add <4 x i32> [[X:%.*]], ; CHECK-NEXT: [[T1:%.*]] = add <4 x i32> [[T0_NEG]], [[Y:%.*]] ; CHECK-NEXT: ret <4 x i32> [[T1]] ; - %t0 = xor <4 x i32> %x, + %t0 = xor <4 x i32> %x, %t1 = sub <4 x i32> %y, %t0 ret <4 x i32> %t1 } diff --git a/llvm/test/Transforms/InstCombine/fpcast.ll b/llvm/test/Transforms/InstCombine/fpcast.ll index d2c932ba447e..69daac773a64 100644 --- a/llvm/test/Transforms/InstCombine/fpcast.ll +++ b/llvm/test/Transforms/InstCombine/fpcast.ll @@ -51,13 +51,13 @@ define half @unary_fneg_fptrunc(float %a) { ret half %c } -define <2 x half> @fneg_fptrunc_vec_undef(<2 x float> %a) { -; CHECK-LABEL: @fneg_fptrunc_vec_undef( +define <2 x half> @fneg_fptrunc_vec_poison(<2 x float> %a) { +; CHECK-LABEL: @fneg_fptrunc_vec_poison( ; CHECK-NEXT: [[TMP1:%.*]] = fptrunc <2 x float> [[A:%.*]] to <2 x half> ; CHECK-NEXT: [[C:%.*]] = fneg <2 x half> [[TMP1]] ; CHECK-NEXT: ret <2 x half> [[C]] ; - %b = fsub <2 x float> , %a + %b = fsub <2 x float> , %a %c = fptrunc <2 x float> %b to <2 x half> ret <2 x half> %c } diff --git a/llvm/test/Transforms/InstCombine/fsub.ll b/llvm/test/Transforms/InstCombine/fsub.ll index 6e13c33b126d..f1e7086e697e 100644 --- a/llvm/test/Transforms/InstCombine/fsub.ll +++ b/llvm/test/Transforms/InstCombine/fsub.ll @@ -153,12 +153,12 @@ define <2 x float> @constant_op1_vec(<2 x float> %x, <2 x float> %y) { ret <2 x float> %r } -define <2 x float> @constant_op1_vec_undef(<2 x float> %x, <2 x float> %y) { -; CHECK-LABEL: @constant_op1_vec_undef( -; CHECK-NEXT: [[R:%.*]] = fadd <2 x float> [[X:%.*]], +define <2 x float> @constant_op1_vec_poison(<2 x float> %x, <2 x float> %y) { +; CHECK-LABEL: @constant_op1_vec_poison( +; CHECK-NEXT: [[R:%.*]] = fadd <2 x float> [[X:%.*]], ; CHECK-NEXT: ret <2 x float> [[R]] ; - %r = fsub <2 x float> %x, + %r = fsub <2 x float> %x, ret <2 x float> %r } @@ -204,12 +204,12 @@ define <2 x float> @unary_neg_op1_vec(<2 x float> %x, <2 x float> %y) { ret <2 x float> %r } -define <2 x float> @neg_op1_vec_undef(<2 x float> %x, <2 x float> %y) { -; CHECK-LABEL: @neg_op1_vec_undef( +define <2 x float> @neg_op1_vec_poison(<2 x float> %x, <2 x float> %y) { +; CHECK-LABEL: @neg_op1_vec_poison( ; CHECK-NEXT: [[R:%.*]] = fadd <2 x float> [[X:%.*]], [[Y:%.*]] ; CHECK-NEXT: ret <2 x float> [[R]] ; - %negy = fsub <2 x float> , %y + %negy = fsub <2 x float> , %y %r = fsub <2 x float> %x, %negy ret <2 x float> %r } diff --git a/llvm/test/Transforms/InstCombine/funnel.ll b/llvm/test/Transforms/InstCombine/funnel.ll index 162519e648f3..a54e6e4642b7 100644 --- a/llvm/test/Transforms/InstCombine/funnel.ll +++ b/llvm/test/Transforms/InstCombine/funnel.ll @@ -43,24 +43,24 @@ define <2 x i16> @fshl_v2i16_constant_splat(<2 x i16> %x, <2 x i16> %y) { ret <2 x i16> %r } -define <2 x i16> @fshl_v2i16_constant_splat_undef0(<2 x i16> %x, <2 x i16> %y) { -; CHECK-LABEL: @fshl_v2i16_constant_splat_undef0( +define <2 x i16> @fshl_v2i16_constant_splat_poison0(<2 x i16> %x, <2 x i16> %y) { +; CHECK-LABEL: @fshl_v2i16_constant_splat_poison0( ; CHECK-NEXT: [[R:%.*]] = call <2 x i16> @llvm.fshl.v2i16(<2 x i16> [[X:%.*]], <2 x i16> [[Y:%.*]], <2 x i16> ) ; CHECK-NEXT: ret <2 x i16> [[R]] ; - %shl = shl <2 x i16> %x, + %shl = shl <2 x i16> %x, %shr = lshr <2 x i16> %y, %r = or <2 x i16> %shl, %shr ret <2 x i16> %r } -define <2 x i16> @fshl_v2i16_constant_splat_undef1(<2 x i16> %x, <2 x i16> %y) { -; CHECK-LABEL: @fshl_v2i16_constant_splat_undef1( +define <2 x i16> @fshl_v2i16_constant_splat_poison1(<2 x i16> %x, <2 x i16> %y) { +; CHECK-LABEL: @fshl_v2i16_constant_splat_poison1( ; CHECK-NEXT: [[R:%.*]] = call <2 x i16> @llvm.fshl.v2i16(<2 x i16> [[X:%.*]], <2 x i16> [[Y:%.*]], <2 x i16> ) ; CHECK-NEXT: ret <2 x i16> [[R]] ; %shl = shl <2 x i16> %x, - %shr = lshr <2 x i16> %y, + %shr = lshr <2 x i16> %y, %r = or <2 x i16> %shl, %shr ret <2 x i16> %r } @@ -78,30 +78,30 @@ define <2 x i17> @fshr_v2i17_constant_splat(<2 x i17> %x, <2 x i17> %y) { ret <2 x i17> %r } -define <2 x i17> @fshr_v2i17_constant_splat_undef0(<2 x i17> %x, <2 x i17> %y) { -; CHECK-LABEL: @fshr_v2i17_constant_splat_undef0( +define <2 x i17> @fshr_v2i17_constant_splat_poison0(<2 x i17> %x, <2 x i17> %y) { +; CHECK-LABEL: @fshr_v2i17_constant_splat_poison0( ; CHECK-NEXT: [[R:%.*]] = call <2 x i17> @llvm.fshl.v2i17(<2 x i17> [[Y:%.*]], <2 x i17> [[X:%.*]], <2 x i17> ) ; CHECK-NEXT: ret <2 x i17> [[R]] ; - %shr = lshr <2 x i17> %x, - %shl = shl <2 x i17> %y, + %shr = lshr <2 x i17> %x, + %shl = shl <2 x i17> %y, %r = or <2 x i17> %shr, %shl ret <2 x i17> %r } -define <2 x i17> @fshr_v2i17_constant_splat_undef1(<2 x i17> %x, <2 x i17> %y) { -; CHECK-LABEL: @fshr_v2i17_constant_splat_undef1( +define <2 x i17> @fshr_v2i17_constant_splat_poison1(<2 x i17> %x, <2 x i17> %y) { +; CHECK-LABEL: @fshr_v2i17_constant_splat_poison1( ; CHECK-NEXT: [[R:%.*]] = call <2 x i17> @llvm.fshl.v2i17(<2 x i17> [[Y:%.*]], <2 x i17> [[X:%.*]], <2 x i17> ) ; CHECK-NEXT: ret <2 x i17> [[R]] ; - %shr = lshr <2 x i17> %x, - %shl = shl <2 x i17> %y, + %shr = lshr <2 x i17> %x, + %shl = shl <2 x i17> %y, %r = or <2 x i17> %shr, %shl ret <2 x i17> %r } ; Allow arbitrary shift constants. -; Support undef elements. +; Support poison elements. define <2 x i32> @fshr_v2i32_constant_nonsplat(<2 x i32> %x, <2 x i32> %y) { ; CHECK-LABEL: @fshr_v2i32_constant_nonsplat( @@ -114,24 +114,24 @@ define <2 x i32> @fshr_v2i32_constant_nonsplat(<2 x i32> %x, <2 x i32> %y) { ret <2 x i32> %r } -define <2 x i32> @fshr_v2i32_constant_nonsplat_undef0(<2 x i32> %x, <2 x i32> %y) { -; CHECK-LABEL: @fshr_v2i32_constant_nonsplat_undef0( +define <2 x i32> @fshr_v2i32_constant_nonsplat_poison0(<2 x i32> %x, <2 x i32> %y) { +; CHECK-LABEL: @fshr_v2i32_constant_nonsplat_poison0( ; CHECK-NEXT: [[R:%.*]] = call <2 x i32> @llvm.fshl.v2i32(<2 x i32> [[Y:%.*]], <2 x i32> [[X:%.*]], <2 x i32> ) ; CHECK-NEXT: ret <2 x i32> [[R]] ; - %shr = lshr <2 x i32> %x, + %shr = lshr <2 x i32> %x, %shl = shl <2 x i32> %y, %r = or <2 x i32> %shl, %shr ret <2 x i32> %r } -define <2 x i32> @fshr_v2i32_constant_nonsplat_undef1(<2 x i32> %x, <2 x i32> %y) { -; CHECK-LABEL: @fshr_v2i32_constant_nonsplat_undef1( -; CHECK-NEXT: [[R:%.*]] = call <2 x i32> @llvm.fshl.v2i32(<2 x i32> [[Y:%.*]], <2 x i32> [[X:%.*]], <2 x i32> ) +define <2 x i32> @fshr_v2i32_constant_nonsplat_poison1(<2 x i32> %x, <2 x i32> %y) { +; CHECK-LABEL: @fshr_v2i32_constant_nonsplat_poison1( +; CHECK-NEXT: [[R:%.*]] = call <2 x i32> @llvm.fshl.v2i32(<2 x i32> [[Y:%.*]], <2 x i32> [[X:%.*]], <2 x i32> ) ; CHECK-NEXT: ret <2 x i32> [[R]] ; %shr = lshr <2 x i32> %x, - %shl = shl <2 x i32> %y, + %shl = shl <2 x i32> %y, %r = or <2 x i32> %shl, %shr ret <2 x i32> %r } @@ -147,13 +147,13 @@ define <2 x i36> @fshl_v2i36_constant_nonsplat(<2 x i36> %x, <2 x i36> %y) { ret <2 x i36> %r } -define <3 x i36> @fshl_v3i36_constant_nonsplat_undef0(<3 x i36> %x, <3 x i36> %y) { -; CHECK-LABEL: @fshl_v3i36_constant_nonsplat_undef0( -; CHECK-NEXT: [[R:%.*]] = call <3 x i36> @llvm.fshl.v3i36(<3 x i36> [[X:%.*]], <3 x i36> [[Y:%.*]], <3 x i36> ) +define <3 x i36> @fshl_v3i36_constant_nonsplat_poison0(<3 x i36> %x, <3 x i36> %y) { +; CHECK-LABEL: @fshl_v3i36_constant_nonsplat_poison0( +; CHECK-NEXT: [[R:%.*]] = call <3 x i36> @llvm.fshl.v3i36(<3 x i36> [[X:%.*]], <3 x i36> [[Y:%.*]], <3 x i36> ) ; CHECK-NEXT: ret <3 x i36> [[R]] ; - %shl = shl <3 x i36> %x, - %shr = lshr <3 x i36> %y, + %shl = shl <3 x i36> %x, + %shr = lshr <3 x i36> %y, %r = or <3 x i36> %shl, %shr ret <3 x i36> %r } diff --git a/llvm/test/Transforms/InstCombine/get-lowbitmask-upto-and-including-bit.ll b/llvm/test/Transforms/InstCombine/get-lowbitmask-upto-and-including-bit.ll index 12a81f0cd2f0..40caa5789136 100644 --- a/llvm/test/Transforms/InstCombine/get-lowbitmask-upto-and-including-bit.ll +++ b/llvm/test/Transforms/InstCombine/get-lowbitmask-upto-and-including-bit.ll @@ -41,36 +41,36 @@ define <2 x i8> @t2_vec(<2 x i8> %x) { %mask = or <2 x i8> %lowbitmask, %bitmask ret <2 x i8> %mask } -define <3 x i8> @t3_vec_undef0(<3 x i8> %x) { -; CHECK-LABEL: @t3_vec_undef0( +define <3 x i8> @t3_vec_poison0(<3 x i8> %x) { +; CHECK-LABEL: @t3_vec_poison0( ; CHECK-NEXT: [[TMP1:%.*]] = sub <3 x i8> , [[X:%.*]] ; CHECK-NEXT: [[MASK:%.*]] = lshr <3 x i8> , [[TMP1]] ; CHECK-NEXT: ret <3 x i8> [[MASK]] ; - %bitmask = shl <3 x i8> , %x + %bitmask = shl <3 x i8> , %x %lowbitmask = add <3 x i8> %bitmask, %mask = or <3 x i8> %lowbitmask, %bitmask ret <3 x i8> %mask } -define <3 x i8> @t4_vec_undef1(<3 x i8> %x) { -; CHECK-LABEL: @t4_vec_undef1( +define <3 x i8> @t4_vec_poison1(<3 x i8> %x) { +; CHECK-LABEL: @t4_vec_poison1( ; CHECK-NEXT: [[TMP1:%.*]] = sub <3 x i8> , [[X:%.*]] ; CHECK-NEXT: [[MASK:%.*]] = lshr <3 x i8> , [[TMP1]] ; CHECK-NEXT: ret <3 x i8> [[MASK]] ; %bitmask = shl <3 x i8> , %x - %lowbitmask = add <3 x i8> %bitmask, + %lowbitmask = add <3 x i8> %bitmask, %mask = or <3 x i8> %lowbitmask, %bitmask ret <3 x i8> %mask } -define <3 x i8> @t5_vec_undef2(<3 x i8> %x) { -; CHECK-LABEL: @t5_vec_undef2( +define <3 x i8> @t5_vec_poison2(<3 x i8> %x) { +; CHECK-LABEL: @t5_vec_poison2( ; CHECK-NEXT: [[TMP1:%.*]] = sub <3 x i8> , [[X:%.*]] ; CHECK-NEXT: [[MASK:%.*]] = lshr <3 x i8> , [[TMP1]] ; CHECK-NEXT: ret <3 x i8> [[MASK]] ; - %bitmask = shl <3 x i8> , %x - %lowbitmask = add <3 x i8> %bitmask, + %bitmask = shl <3 x i8> , %x + %lowbitmask = add <3 x i8> %bitmask, %mask = or <3 x i8> %lowbitmask, %bitmask ret <3 x i8> %mask } diff --git a/llvm/test/Transforms/InstCombine/hoist-negation-out-of-bias-calculation.ll b/llvm/test/Transforms/InstCombine/hoist-negation-out-of-bias-calculation.ll index c8f14595ea67..e4cae1351978 100644 --- a/llvm/test/Transforms/InstCombine/hoist-negation-out-of-bias-calculation.ll +++ b/llvm/test/Transforms/InstCombine/hoist-negation-out-of-bias-calculation.ll @@ -55,14 +55,14 @@ define <2 x i8> @t2_vec(<2 x i8> %x, <2 x i8> %y) { ret <2 x i8> %negbias } -define <2 x i8> @t3_vec_undef(<2 x i8> %x, <2 x i8> %y) { -; CHECK-LABEL: @t3_vec_undef( +define <2 x i8> @t3_vec_poison(<2 x i8> %x, <2 x i8> %y) { +; CHECK-LABEL: @t3_vec_poison( ; CHECK-NEXT: [[TMP1:%.*]] = add <2 x i8> [[Y:%.*]], ; CHECK-NEXT: [[TMP2:%.*]] = and <2 x i8> [[TMP1]], [[X:%.*]] ; CHECK-NEXT: [[NEGBIAS:%.*]] = sub <2 x i8> zeroinitializer, [[TMP2]] ; CHECK-NEXT: ret <2 x i8> [[NEGBIAS]] ; - %negy = sub <2 x i8> , %y + %negy = sub <2 x i8> , %y %unbiasedx = and <2 x i8> %negy, %x %negbias = sub <2 x i8> %unbiasedx, %x ret <2 x i8> %negbias diff --git a/llvm/test/Transforms/InstCombine/hoist-not-from-ashr-operand.ll b/llvm/test/Transforms/InstCombine/hoist-not-from-ashr-operand.ll index e0242855e268..2217666f0f49 100644 --- a/llvm/test/Transforms/InstCombine/hoist-not-from-ashr-operand.ll +++ b/llvm/test/Transforms/InstCombine/hoist-not-from-ashr-operand.ll @@ -41,14 +41,14 @@ define <2 x i8> @t2_vec(<2 x i8> %x, <2 x i8> %y) { %ashr = ashr <2 x i8> %not_x, %y ret <2 x i8> %ashr } -; Note that we must sanitize undef elts of -1 constant to -1 or 0. -define <2 x i8> @t3_vec_undef(<2 x i8> %x, <2 x i8> %y) { -; CHECK-LABEL: @t3_vec_undef( +; Note that we must sanitize poison elts of -1 constant to -1 or 0. +define <2 x i8> @t3_vec_poison(<2 x i8> %x, <2 x i8> %y) { +; CHECK-LABEL: @t3_vec_poison( ; CHECK-NEXT: [[NOT_X_NOT:%.*]] = ashr <2 x i8> [[X:%.*]], [[Y:%.*]] ; CHECK-NEXT: [[ASHR:%.*]] = xor <2 x i8> [[NOT_X_NOT]], ; CHECK-NEXT: ret <2 x i8> [[ASHR]] ; - %not_x = xor <2 x i8> %x, + %not_x = xor <2 x i8> %x, %ashr = ashr <2 x i8> %not_x, %y ret <2 x i8> %ashr } diff --git a/llvm/test/Transforms/InstCombine/icmp-uge-of-add-of-shl-one-by-bits-to-allones-and-val-to-icmp-eq-of-lshr-val-by-bits-and-0.ll b/llvm/test/Transforms/InstCombine/icmp-uge-of-add-of-shl-one-by-bits-to-allones-and-val-to-icmp-eq-of-lshr-val-by-bits-and-0.ll index 5adf476f7a79..32ef6267cdf8 100644 --- a/llvm/test/Transforms/InstCombine/icmp-uge-of-add-of-shl-one-by-bits-to-allones-and-val-to-icmp-eq-of-lshr-val-by-bits-and-0.ll +++ b/llvm/test/Transforms/InstCombine/icmp-uge-of-add-of-shl-one-by-bits-to-allones-and-val-to-icmp-eq-of-lshr-val-by-bits-and-0.ll @@ -56,8 +56,8 @@ define <3 x i1> @p2_vec_undef0(<3 x i8> %val, <3 x i8> %bits) { ; CHECK-LABEL: @p2_vec_undef0( ; CHECK-NEXT: [[T0:%.*]] = shl <3 x i8> , [[BITS:%.*]] ; CHECK-NEXT: call void @use3i8(<3 x i8> [[T0]]) -; CHECK-NEXT: [[VAL_HIGHBITS:%.*]] = lshr <3 x i8> [[VAL:%.*]], [[BITS]] -; CHECK-NEXT: [[R:%.*]] = icmp eq <3 x i8> [[VAL_HIGHBITS]], zeroinitializer +; CHECK-NEXT: [[T1:%.*]] = add <3 x i8> [[T0]], +; CHECK-NEXT: [[R:%.*]] = icmp uge <3 x i8> [[T1]], [[VAL:%.*]] ; CHECK-NEXT: ret <3 x i1> [[R]] ; %t0 = shl <3 x i8> , %bits diff --git a/llvm/test/Transforms/InstCombine/icmp-uge-of-not-of-shl-allones-by-bits-and-val-to-icmp-eq-of-lshr-val-by-bits-and-0.ll b/llvm/test/Transforms/InstCombine/icmp-uge-of-not-of-shl-allones-by-bits-and-val-to-icmp-eq-of-lshr-val-by-bits-and-0.ll index 7f4603881f23..27b02c8c6e93 100644 --- a/llvm/test/Transforms/InstCombine/icmp-uge-of-not-of-shl-allones-by-bits-and-val-to-icmp-eq-of-lshr-val-by-bits-and-0.ll +++ b/llvm/test/Transforms/InstCombine/icmp-uge-of-not-of-shl-allones-by-bits-and-val-to-icmp-eq-of-lshr-val-by-bits-and-0.ll @@ -40,38 +40,38 @@ define <2 x i1> @p1_vec(<2 x i8> %val, <2 x i8> %bits) { ret <2 x i1> %r } -define <3 x i1> @p2_vec_undef0(<3 x i8> %val, <3 x i8> %bits) { -; CHECK-LABEL: @p2_vec_undef0( +define <3 x i1> @p2_vec_poison0(<3 x i8> %val, <3 x i8> %bits) { +; CHECK-LABEL: @p2_vec_poison0( ; CHECK-NEXT: [[VAL_HIGHBITS:%.*]] = lshr <3 x i8> [[VAL:%.*]], [[BITS:%.*]] ; CHECK-NEXT: [[R:%.*]] = icmp eq <3 x i8> [[VAL_HIGHBITS]], zeroinitializer ; CHECK-NEXT: ret <3 x i1> [[R]] ; - %t0 = shl <3 x i8> , %bits + %t0 = shl <3 x i8> , %bits %t1 = xor <3 x i8> %t0, %r = icmp uge <3 x i8> %t1, %val ret <3 x i1> %r } -define <3 x i1> @p2_vec_undef1(<3 x i8> %val, <3 x i8> %bits) { -; CHECK-LABEL: @p2_vec_undef1( +define <3 x i1> @p2_vec_poison1(<3 x i8> %val, <3 x i8> %bits) { +; CHECK-LABEL: @p2_vec_poison1( ; CHECK-NEXT: [[VAL_HIGHBITS:%.*]] = lshr <3 x i8> [[VAL:%.*]], [[BITS:%.*]] ; CHECK-NEXT: [[R:%.*]] = icmp eq <3 x i8> [[VAL_HIGHBITS]], zeroinitializer ; CHECK-NEXT: ret <3 x i1> [[R]] ; %t0 = shl <3 x i8> , %bits - %t1 = xor <3 x i8> %t0, + %t1 = xor <3 x i8> %t0, %r = icmp uge <3 x i8> %t1, %val ret <3 x i1> %r } -define <3 x i1> @p2_vec_undef2(<3 x i8> %val, <3 x i8> %bits) { -; CHECK-LABEL: @p2_vec_undef2( +define <3 x i1> @p2_vec_poison2(<3 x i8> %val, <3 x i8> %bits) { +; CHECK-LABEL: @p2_vec_poison2( ; CHECK-NEXT: [[VAL_HIGHBITS:%.*]] = lshr <3 x i8> [[VAL:%.*]], [[BITS:%.*]] ; CHECK-NEXT: [[R:%.*]] = icmp eq <3 x i8> [[VAL_HIGHBITS]], zeroinitializer ; CHECK-NEXT: ret <3 x i1> [[R]] ; - %t0 = shl <3 x i8> , %bits - %t1 = xor <3 x i8> %t0, + %t0 = shl <3 x i8> , %bits + %t1 = xor <3 x i8> %t0, %r = icmp uge <3 x i8> %t1, %val ret <3 x i1> %r } diff --git a/llvm/test/Transforms/InstCombine/icmp-ugt-of-shl-1-by-bits-and-val-to-icmp-eq-of-lshr-val-by-bits-and-0.ll b/llvm/test/Transforms/InstCombine/icmp-ugt-of-shl-1-by-bits-and-val-to-icmp-eq-of-lshr-val-by-bits-and-0.ll index 550e8bb17229..72cfb5a9f8bd 100644 --- a/llvm/test/Transforms/InstCombine/icmp-ugt-of-shl-1-by-bits-and-val-to-icmp-eq-of-lshr-val-by-bits-and-0.ll +++ b/llvm/test/Transforms/InstCombine/icmp-ugt-of-shl-1-by-bits-and-val-to-icmp-eq-of-lshr-val-by-bits-and-0.ll @@ -38,13 +38,13 @@ define <2 x i1> @p1_vec(<2 x i8> %val, <2 x i8> %bits) { ret <2 x i1> %r } -define <3 x i1> @p2_vec_undef(<3 x i8> %val, <3 x i8> %bits) { -; CHECK-LABEL: @p2_vec_undef( +define <3 x i1> @p2_vec_poison(<3 x i8> %val, <3 x i8> %bits) { +; CHECK-LABEL: @p2_vec_poison( ; CHECK-NEXT: [[VAL_HIGHBITS:%.*]] = lshr <3 x i8> [[VAL:%.*]], [[BITS:%.*]] ; CHECK-NEXT: [[R:%.*]] = icmp eq <3 x i8> [[VAL_HIGHBITS]], zeroinitializer ; CHECK-NEXT: ret <3 x i1> [[R]] ; - %t0 = shl <3 x i8> , %bits + %t0 = shl <3 x i8> , %bits %r = icmp ugt <3 x i8> %t0, %val ret <3 x i1> %r } diff --git a/llvm/test/Transforms/InstCombine/icmp-ule-of-shl-1-by-bits-and-val-to-icmp-ne-of-lshr-val-by-bits-and-0.ll b/llvm/test/Transforms/InstCombine/icmp-ule-of-shl-1-by-bits-and-val-to-icmp-ne-of-lshr-val-by-bits-and-0.ll index 26b667d36728..79e6914f0953 100644 --- a/llvm/test/Transforms/InstCombine/icmp-ule-of-shl-1-by-bits-and-val-to-icmp-ne-of-lshr-val-by-bits-and-0.ll +++ b/llvm/test/Transforms/InstCombine/icmp-ule-of-shl-1-by-bits-and-val-to-icmp-ne-of-lshr-val-by-bits-and-0.ll @@ -38,13 +38,13 @@ define <2 x i1> @p1_vec(<2 x i8> %val, <2 x i8> %bits) { ret <2 x i1> %r } -define <3 x i1> @p2_vec_undef(<3 x i8> %val, <3 x i8> %bits) { -; CHECK-LABEL: @p2_vec_undef( +define <3 x i1> @p2_vec_poison(<3 x i8> %val, <3 x i8> %bits) { +; CHECK-LABEL: @p2_vec_poison( ; CHECK-NEXT: [[VAL_HIGHBITS:%.*]] = lshr <3 x i8> [[VAL:%.*]], [[BITS:%.*]] ; CHECK-NEXT: [[R:%.*]] = icmp ne <3 x i8> [[VAL_HIGHBITS]], zeroinitializer ; CHECK-NEXT: ret <3 x i1> [[R]] ; - %t0 = shl <3 x i8> , %bits + %t0 = shl <3 x i8> , %bits %r = icmp ule <3 x i8> %t0, %val ret <3 x i1> %r } diff --git a/llvm/test/Transforms/InstCombine/icmp-ult-of-add-of-shl-one-by-bits-to-allones-and-val-to-icmp-ne-of-lshr-val-by-bits-and-0.ll b/llvm/test/Transforms/InstCombine/icmp-ult-of-add-of-shl-one-by-bits-to-allones-and-val-to-icmp-ne-of-lshr-val-by-bits-and-0.ll index dd353d44218b..25894a22f007 100644 --- a/llvm/test/Transforms/InstCombine/icmp-ult-of-add-of-shl-one-by-bits-to-allones-and-val-to-icmp-ne-of-lshr-val-by-bits-and-0.ll +++ b/llvm/test/Transforms/InstCombine/icmp-ult-of-add-of-shl-one-by-bits-to-allones-and-val-to-icmp-ne-of-lshr-val-by-bits-and-0.ll @@ -56,8 +56,8 @@ define <3 x i1> @p2_vec_undef0(<3 x i8> %val, <3 x i8> %bits) { ; CHECK-LABEL: @p2_vec_undef0( ; CHECK-NEXT: [[T0:%.*]] = shl <3 x i8> , [[BITS:%.*]] ; CHECK-NEXT: call void @use3i8(<3 x i8> [[T0]]) -; CHECK-NEXT: [[VAL_HIGHBITS:%.*]] = lshr <3 x i8> [[VAL:%.*]], [[BITS]] -; CHECK-NEXT: [[R:%.*]] = icmp ne <3 x i8> [[VAL_HIGHBITS]], zeroinitializer +; CHECK-NEXT: [[T1:%.*]] = add <3 x i8> [[T0]], +; CHECK-NEXT: [[R:%.*]] = icmp ult <3 x i8> [[T1]], [[VAL:%.*]] ; CHECK-NEXT: ret <3 x i1> [[R]] ; %t0 = shl <3 x i8> , %bits diff --git a/llvm/test/Transforms/InstCombine/icmp-ult-of-not-of-shl-allones-by-bits-and-val-to-icmp-ne-of-lshr-val-by-bits-and-0.ll b/llvm/test/Transforms/InstCombine/icmp-ult-of-not-of-shl-allones-by-bits-and-val-to-icmp-ne-of-lshr-val-by-bits-and-0.ll index c7a45c5cdc11..8441033d4857 100644 --- a/llvm/test/Transforms/InstCombine/icmp-ult-of-not-of-shl-allones-by-bits-and-val-to-icmp-ne-of-lshr-val-by-bits-and-0.ll +++ b/llvm/test/Transforms/InstCombine/icmp-ult-of-not-of-shl-allones-by-bits-and-val-to-icmp-ne-of-lshr-val-by-bits-and-0.ll @@ -40,38 +40,38 @@ define <2 x i1> @p1_vec(<2 x i8> %val, <2 x i8> %bits) { ret <2 x i1> %r } -define <3 x i1> @p2_vec_undef0(<3 x i8> %val, <3 x i8> %bits) { -; CHECK-LABEL: @p2_vec_undef0( +define <3 x i1> @p2_vec_poison0(<3 x i8> %val, <3 x i8> %bits) { +; CHECK-LABEL: @p2_vec_poison0( ; CHECK-NEXT: [[VAL_HIGHBITS:%.*]] = lshr <3 x i8> [[VAL:%.*]], [[BITS:%.*]] ; CHECK-NEXT: [[R:%.*]] = icmp ne <3 x i8> [[VAL_HIGHBITS]], zeroinitializer ; CHECK-NEXT: ret <3 x i1> [[R]] ; - %t0 = shl <3 x i8> , %bits + %t0 = shl <3 x i8> , %bits %t1 = xor <3 x i8> %t0, %r = icmp ult <3 x i8> %t1, %val ret <3 x i1> %r } -define <3 x i1> @p2_vec_undef1(<3 x i8> %val, <3 x i8> %bits) { -; CHECK-LABEL: @p2_vec_undef1( +define <3 x i1> @p2_vec_poison1(<3 x i8> %val, <3 x i8> %bits) { +; CHECK-LABEL: @p2_vec_poison1( ; CHECK-NEXT: [[VAL_HIGHBITS:%.*]] = lshr <3 x i8> [[VAL:%.*]], [[BITS:%.*]] ; CHECK-NEXT: [[R:%.*]] = icmp ne <3 x i8> [[VAL_HIGHBITS]], zeroinitializer ; CHECK-NEXT: ret <3 x i1> [[R]] ; %t0 = shl <3 x i8> , %bits - %t1 = xor <3 x i8> %t0, + %t1 = xor <3 x i8> %t0, %r = icmp ult <3 x i8> %t1, %val ret <3 x i1> %r } -define <3 x i1> @p2_vec_undef2(<3 x i8> %val, <3 x i8> %bits) { -; CHECK-LABEL: @p2_vec_undef2( +define <3 x i1> @p2_vec_poison2(<3 x i8> %val, <3 x i8> %bits) { +; CHECK-LABEL: @p2_vec_poison2( ; CHECK-NEXT: [[VAL_HIGHBITS:%.*]] = lshr <3 x i8> [[VAL:%.*]], [[BITS:%.*]] ; CHECK-NEXT: [[R:%.*]] = icmp ne <3 x i8> [[VAL_HIGHBITS]], zeroinitializer ; CHECK-NEXT: ret <3 x i1> [[R]] ; - %t0 = shl <3 x i8> , %bits - %t1 = xor <3 x i8> %t0, + %t0 = shl <3 x i8> , %bits + %t1 = xor <3 x i8> %t0, %r = icmp ult <3 x i8> %t1, %val ret <3 x i1> %r } diff --git a/llvm/test/Transforms/InstCombine/icmp.ll b/llvm/test/Transforms/InstCombine/icmp.ll index 10ab1fe11834..31093c7ca103 100644 --- a/llvm/test/Transforms/InstCombine/icmp.ll +++ b/llvm/test/Transforms/InstCombine/icmp.ll @@ -1790,14 +1790,14 @@ define <2 x i1> @icmp_add20_eq_add57_splat(<2 x i32> %x, <2 x i32> %y) { ret <2 x i1> %cmp } -define <2 x i1> @icmp_add20_eq_add57_undef(<2 x i32> %x, <2 x i32> %y) { -; CHECK-LABEL: @icmp_add20_eq_add57_undef( +define <2 x i1> @icmp_add20_eq_add57_poison(<2 x i32> %x, <2 x i32> %y) { +; CHECK-LABEL: @icmp_add20_eq_add57_poison( ; CHECK-NEXT: [[TMP1:%.*]] = add <2 x i32> [[Y:%.*]], ; CHECK-NEXT: [[CMP:%.*]] = icmp eq <2 x i32> [[TMP1]], [[X:%.*]] ; CHECK-NEXT: ret <2 x i1> [[CMP]] ; %1 = add <2 x i32> %x, - %2 = add <2 x i32> %y, + %2 = add <2 x i32> %y, %cmp = icmp eq <2 x i32> %1, %2 ret <2 x i1> %cmp } @@ -1838,14 +1838,14 @@ define <2 x i1> @icmp_sub57_ne_sub20_splat(<2 x i32> %x, <2 x i32> %y) { ret <2 x i1> %cmp } -define <2 x i1> @icmp_sub57_ne_sub20_vec_undef(<2 x i32> %x, <2 x i32> %y) { -; CHECK-LABEL: @icmp_sub57_ne_sub20_vec_undef( +define <2 x i1> @icmp_sub57_ne_sub20_vec_poison(<2 x i32> %x, <2 x i32> %y) { +; CHECK-LABEL: @icmp_sub57_ne_sub20_vec_poison( ; CHECK-NEXT: [[TMP1:%.*]] = add <2 x i32> [[X:%.*]], ; CHECK-NEXT: [[CMP:%.*]] = icmp ne <2 x i32> [[TMP1]], [[Y:%.*]] ; CHECK-NEXT: ret <2 x i1> [[CMP]] ; - %1 = add <2 x i32> %x, - %2 = add <2 x i32> %y, + %1 = add <2 x i32> %x, + %2 = add <2 x i32> %y, %cmp = icmp ne <2 x i32> %1, %2 ret <2 x i1> %cmp } @@ -1926,14 +1926,14 @@ define <2 x i1> @icmp_add20_sge_add57_splat(<2 x i32> %x, <2 x i32> %y) { ret <2 x i1> %cmp } -define <2 x i1> @icmp_add20_sge_add57_undef(<2 x i32> %x, <2 x i32> %y) { -; CHECK-LABEL: @icmp_add20_sge_add57_undef( +define <2 x i1> @icmp_add20_sge_add57_poison(<2 x i32> %x, <2 x i32> %y) { +; CHECK-LABEL: @icmp_add20_sge_add57_poison( ; CHECK-NEXT: [[TMP1:%.*]] = add nsw <2 x i32> [[Y:%.*]], ; CHECK-NEXT: [[CMP:%.*]] = icmp sle <2 x i32> [[TMP1]], [[X:%.*]] ; CHECK-NEXT: ret <2 x i1> [[CMP]] ; %1 = add nsw <2 x i32> %x, - %2 = add nsw <2 x i32> %y, + %2 = add nsw <2 x i32> %y, %cmp = icmp sge <2 x i32> %1, %2 ret <2 x i1> %cmp } @@ -1975,14 +1975,14 @@ define <2 x i1> @icmp_sub57_sge_sub20_splat(<2 x i32> %x, <2 x i32> %y) { ret <2 x i1> %cmp } -define <2 x i1> @icmp_sub57_sge_sub20_vec_undef(<2 x i32> %x, <2 x i32> %y) { -; CHECK-LABEL: @icmp_sub57_sge_sub20_vec_undef( +define <2 x i1> @icmp_sub57_sge_sub20_vec_poison(<2 x i32> %x, <2 x i32> %y) { +; CHECK-LABEL: @icmp_sub57_sge_sub20_vec_poison( ; CHECK-NEXT: [[TMP1:%.*]] = add nsw <2 x i32> [[X:%.*]], ; CHECK-NEXT: [[CMP:%.*]] = icmp sge <2 x i32> [[TMP1]], [[Y:%.*]] ; CHECK-NEXT: ret <2 x i1> [[CMP]] ; - %1 = add nsw <2 x i32> %x, - %2 = add nsw <2 x i32> %y, + %1 = add nsw <2 x i32> %x, + %2 = add nsw <2 x i32> %y, %cmp = icmp sge <2 x i32> %1, %2 ret <2 x i1> %cmp } @@ -2557,13 +2557,13 @@ define <2 x i1> @or_icmp_eq_B_0_icmp_ult_A_B_uniform(<2 x i64> %a, <2 x i64> %b) ret <2 x i1> %3 } -define <2 x i1> @or_icmp_eq_B_0_icmp_ult_A_B_undef(<2 x i64> %a, <2 x i64> %b) { -; CHECK-LABEL: @or_icmp_eq_B_0_icmp_ult_A_B_undef( +define <2 x i1> @or_icmp_eq_B_0_icmp_ult_A_B_poison(<2 x i64> %a, <2 x i64> %b) { +; CHECK-LABEL: @or_icmp_eq_B_0_icmp_ult_A_B_poison( ; CHECK-NEXT: [[TMP1:%.*]] = add <2 x i64> [[B:%.*]], ; CHECK-NEXT: [[TMP2:%.*]] = icmp uge <2 x i64> [[TMP1]], [[A:%.*]] ; CHECK-NEXT: ret <2 x i1> [[TMP2]] ; - %1 = icmp eq <2 x i64> %b, + %1 = icmp eq <2 x i64> %b, %2 = icmp ult <2 x i64> %a, %b %3 = or <2 x i1> %1, %2 ret <2 x i1> %3 @@ -2606,14 +2606,14 @@ define <2 x i1> @or_icmp_ne_A_0_icmp_ne_B_0_uniform(<2 x i64> %a, <2 x i64> %b) ret <2 x i1> %3 } -define <2 x i1> @or_icmp_ne_A_0_icmp_ne_B_0_undef(<2 x i64> %a, <2 x i64> %b) { -; CHECK-LABEL: @or_icmp_ne_A_0_icmp_ne_B_0_undef( +define <2 x i1> @or_icmp_ne_A_0_icmp_ne_B_0_poison(<2 x i64> %a, <2 x i64> %b) { +; CHECK-LABEL: @or_icmp_ne_A_0_icmp_ne_B_0_poison( ; CHECK-NEXT: [[TMP1:%.*]] = or <2 x i64> [[A:%.*]], [[B:%.*]] ; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <2 x i64> [[TMP1]], zeroinitializer ; CHECK-NEXT: ret <2 x i1> [[TMP2]] ; - %1 = icmp ne <2 x i64> %a, - %2 = icmp ne <2 x i64> %b, + %1 = icmp ne <2 x i64> %a, + %2 = icmp ne <2 x i64> %b, %3 = or <2 x i1> %1, %2 ret <2 x i1> %3 } @@ -2803,13 +2803,13 @@ define <2 x i1> @and_icmp_ne_B_0_icmp_uge_A_B_uniform(<2 x i64> %a, <2 x i64> %b ret <2 x i1> %3 } -define <2 x i1> @and_icmp_ne_B_0_icmp_uge_A_B_undef(<2 x i64> %a, <2 x i64> %b) { -; CHECK-LABEL: @and_icmp_ne_B_0_icmp_uge_A_B_undef( +define <2 x i1> @and_icmp_ne_B_0_icmp_uge_A_B_poison(<2 x i64> %a, <2 x i64> %b) { +; CHECK-LABEL: @and_icmp_ne_B_0_icmp_uge_A_B_poison( ; CHECK-NEXT: [[TMP1:%.*]] = add <2 x i64> [[B:%.*]], ; CHECK-NEXT: [[TMP2:%.*]] = icmp ult <2 x i64> [[TMP1]], [[A:%.*]] ; CHECK-NEXT: ret <2 x i1> [[TMP2]] ; - %1 = icmp ne <2 x i64> %b, + %1 = icmp ne <2 x i64> %b, %2 = icmp uge <2 x i64> %a, %b %3 = and <2 x i1> %1, %2 ret <2 x i1> %3 @@ -3272,13 +3272,13 @@ define <2 x i1> @icmp_and_or_lshr_cst_vec_nonuniform(<2 x i32> %x) { ret <2 x i1> %ret } -define <2 x i1> @icmp_and_or_lshr_cst_vec_undef(<2 x i32> %x) { -; CHECK-LABEL: @icmp_and_or_lshr_cst_vec_undef( +define <2 x i1> @icmp_and_or_lshr_cst_vec_poison(<2 x i32> %x) { +; CHECK-LABEL: @icmp_and_or_lshr_cst_vec_poison( ; CHECK-NEXT: [[TMP1:%.*]] = and <2 x i32> [[X:%.*]], ; CHECK-NEXT: [[RET:%.*]] = icmp ne <2 x i32> [[TMP1]], zeroinitializer ; CHECK-NEXT: ret <2 x i1> [[RET]] ; - %shf = lshr <2 x i32> %x, + %shf = lshr <2 x i32> %x, %or = or <2 x i32> %shf, %x %and = and <2 x i32> %or, %ret = icmp ne <2 x i32> %and, zeroinitializer @@ -3315,15 +3315,15 @@ define <2 x i1> @icmp_and_or_lshr_cst_vec_nonuniform_commute(<2 x i32> %xp) { ret <2 x i1> %ret } -define <2 x i1> @icmp_and_or_lshr_cst_vec_undef_commute(<2 x i32> %xp) { -; CHECK-LABEL: @icmp_and_or_lshr_cst_vec_undef_commute( +define <2 x i1> @icmp_and_or_lshr_cst_vec_poison_commute(<2 x i32> %xp) { +; CHECK-LABEL: @icmp_and_or_lshr_cst_vec_poison_commute( ; CHECK-NEXT: [[X:%.*]] = srem <2 x i32> [[XP:%.*]], ; CHECK-NEXT: [[TMP1:%.*]] = and <2 x i32> [[X]], ; CHECK-NEXT: [[RET:%.*]] = icmp ne <2 x i32> [[TMP1]], zeroinitializer ; CHECK-NEXT: ret <2 x i1> [[RET]] ; %x = srem <2 x i32> %xp, ; prevent complexity-based canonicalization - %shf = lshr <2 x i32> %x, + %shf = lshr <2 x i32> %x, %or = or <2 x i32> %x, %shf %and = and <2 x i32> %or, %ret = icmp ne <2 x i32> %and, zeroinitializer @@ -4360,7 +4360,7 @@ define <2 x i1> @signbit_false_logic(<2 x i5> %x) { ; CHECK-NEXT: [[R:%.*]] = icmp ne <2 x i5> [[X:%.*]], zeroinitializer ; CHECK-NEXT: ret <2 x i1> [[R]] ; - %dec = add <2 x i5> %x, + %dec = add <2 x i5> %x, %not = xor <2 x i5> %x, %and = and <2 x i5> %dec, %not %r = icmp sgt <2 x i5> %and, diff --git a/llvm/test/Transforms/InstCombine/integer-round-up-pow2-alignment.ll b/llvm/test/Transforms/InstCombine/integer-round-up-pow2-alignment.ll index 7cef922eaf0c..c7e0553992b9 100644 --- a/llvm/test/Transforms/InstCombine/integer-round-up-pow2-alignment.ll +++ b/llvm/test/Transforms/InstCombine/integer-round-up-pow2-alignment.ll @@ -86,9 +86,9 @@ define <2 x i8> @t4_splat(<2 x i8> %x) { ret <2 x i8> %x.roundedup } -; Splat-with-undef -define <2 x i8> @t5_splat_undef_0b0001(<2 x i8> %x) { -; CHECK-LABEL: @t5_splat_undef_0b0001( +; Splat-with-poison +define <2 x i8> @t5_splat_poison_0b0001(<2 x i8> %x) { +; CHECK-LABEL: @t5_splat_poison_0b0001( ; CHECK-NEXT: [[X_BIASED1:%.*]] = add <2 x i8> [[X:%.*]], ; CHECK-NEXT: [[X_ROUNDEDUP:%.*]] = and <2 x i8> [[X_BIASED1]], ; CHECK-NEXT: ret <2 x i8> [[X_ROUNDEDUP]] @@ -96,43 +96,43 @@ define <2 x i8> @t5_splat_undef_0b0001(<2 x i8> %x) { %x.lowbits = and <2 x i8> %x, %x.lowbits.are.zero = icmp eq <2 x i8> %x.lowbits, %x.biased = add <2 x i8> %x, - %x.biased.highbits = and <2 x i8> %x.biased, + %x.biased.highbits = and <2 x i8> %x.biased, %x.roundedup = select <2 x i1> %x.lowbits.are.zero, <2 x i8> %x, <2 x i8> %x.biased.highbits ret <2 x i8> %x.roundedup } -define <2 x i8> @t5_splat_undef_0b0010(<2 x i8> %x) { -; CHECK-LABEL: @t5_splat_undef_0b0010( +define <2 x i8> @t5_splat_poison_0b0010(<2 x i8> %x) { +; CHECK-LABEL: @t5_splat_poison_0b0010( ; CHECK-NEXT: [[X_BIASED1:%.*]] = add <2 x i8> [[X:%.*]], ; CHECK-NEXT: [[X_ROUNDEDUP:%.*]] = and <2 x i8> [[X_BIASED1]], ; CHECK-NEXT: ret <2 x i8> [[X_ROUNDEDUP]] ; %x.lowbits = and <2 x i8> %x, %x.lowbits.are.zero = icmp eq <2 x i8> %x.lowbits, - %x.biased = add <2 x i8> %x, + %x.biased = add <2 x i8> %x, %x.biased.highbits = and <2 x i8> %x.biased, %x.roundedup = select <2 x i1> %x.lowbits.are.zero, <2 x i8> %x, <2 x i8> %x.biased.highbits ret <2 x i8> %x.roundedup } -define <2 x i8> @t5_splat_undef_0b0100(<2 x i8> %x) { -; CHECK-LABEL: @t5_splat_undef_0b0100( +define <2 x i8> @t5_splat_poison_0b0100(<2 x i8> %x) { +; CHECK-LABEL: @t5_splat_poison_0b0100( ; CHECK-NEXT: [[X_BIASED:%.*]] = add <2 x i8> [[X:%.*]], ; CHECK-NEXT: [[X_ROUNDEDUP:%.*]] = and <2 x i8> [[X_BIASED]], ; CHECK-NEXT: ret <2 x i8> [[X_ROUNDEDUP]] ; %x.lowbits = and <2 x i8> %x, - %x.lowbits.are.zero = icmp eq <2 x i8> %x.lowbits, + %x.lowbits.are.zero = icmp eq <2 x i8> %x.lowbits, %x.biased = add <2 x i8> %x, %x.biased.highbits = and <2 x i8> %x.biased, %x.roundedup = select <2 x i1> %x.lowbits.are.zero, <2 x i8> %x, <2 x i8> %x.biased.highbits ret <2 x i8> %x.roundedup } -define <2 x i8> @t5_splat_undef_0b1000(<2 x i8> %x) { -; CHECK-LABEL: @t5_splat_undef_0b1000( +define <2 x i8> @t5_splat_poison_0b1000(<2 x i8> %x) { +; CHECK-LABEL: @t5_splat_poison_0b1000( ; CHECK-NEXT: [[X_BIASED:%.*]] = add <2 x i8> [[X:%.*]], ; CHECK-NEXT: [[X_ROUNDEDUP:%.*]] = and <2 x i8> [[X_BIASED]], ; CHECK-NEXT: ret <2 x i8> [[X_ROUNDEDUP]] ; - %x.lowbits = and <2 x i8> %x, + %x.lowbits = and <2 x i8> %x, %x.lowbits.are.zero = icmp eq <2 x i8> %x.lowbits, %x.biased = add <2 x i8> %x, %x.biased.highbits = and <2 x i8> %x.biased, @@ -177,64 +177,64 @@ define <2 x i8> @t7_nonsplat_bias(<2 x i8> %x) { } ; Splat-in-disguise vector tests -define <2 x i8> @t8_nonsplat_masked_by_undef_0b0001(<2 x i8> %x) { -; CHECK-LABEL: @t8_nonsplat_masked_by_undef_0b0001( +define <2 x i8> @t8_nonsplat_masked_by_poison_0b0001(<2 x i8> %x) { +; CHECK-LABEL: @t8_nonsplat_masked_by_poison_0b0001( ; CHECK-NEXT: [[X_LOWBITS:%.*]] = and <2 x i8> [[X:%.*]], ; CHECK-NEXT: [[X_LOWBITS_ARE_ZERO:%.*]] = icmp eq <2 x i8> [[X_LOWBITS]], zeroinitializer ; CHECK-NEXT: [[X_BIASED:%.*]] = add <2 x i8> [[X]], -; CHECK-NEXT: [[X_BIASED_HIGHBITS:%.*]] = and <2 x i8> [[X_BIASED]], +; CHECK-NEXT: [[X_BIASED_HIGHBITS:%.*]] = and <2 x i8> [[X_BIASED]], ; CHECK-NEXT: [[X_ROUNDEDUP:%.*]] = select <2 x i1> [[X_LOWBITS_ARE_ZERO]], <2 x i8> [[X]], <2 x i8> [[X_BIASED_HIGHBITS]] ; CHECK-NEXT: ret <2 x i8> [[X_ROUNDEDUP]] ; %x.lowbits = and <2 x i8> %x, %x.lowbits.are.zero = icmp eq <2 x i8> %x.lowbits, %x.biased = add <2 x i8> %x, - %x.biased.highbits = and <2 x i8> %x.biased, + %x.biased.highbits = and <2 x i8> %x.biased, %x.roundedup = select <2 x i1> %x.lowbits.are.zero, <2 x i8> %x, <2 x i8> %x.biased.highbits ret <2 x i8> %x.roundedup } -define <2 x i8> @t8_nonsplat_masked_by_undef_0b0010(<2 x i8> %x) { -; CHECK-LABEL: @t8_nonsplat_masked_by_undef_0b0010( +define <2 x i8> @t8_nonsplat_masked_by_poison_0b0010(<2 x i8> %x) { +; CHECK-LABEL: @t8_nonsplat_masked_by_poison_0b0010( ; CHECK-NEXT: [[X_LOWBITS:%.*]] = and <2 x i8> [[X:%.*]], ; CHECK-NEXT: [[X_LOWBITS_ARE_ZERO:%.*]] = icmp eq <2 x i8> [[X_LOWBITS]], zeroinitializer -; CHECK-NEXT: [[X_BIASED:%.*]] = add <2 x i8> [[X]], +; CHECK-NEXT: [[X_BIASED:%.*]] = add <2 x i8> [[X]], ; CHECK-NEXT: [[X_BIASED_HIGHBITS:%.*]] = and <2 x i8> [[X_BIASED]], ; CHECK-NEXT: [[X_ROUNDEDUP:%.*]] = select <2 x i1> [[X_LOWBITS_ARE_ZERO]], <2 x i8> [[X]], <2 x i8> [[X_BIASED_HIGHBITS]] ; CHECK-NEXT: ret <2 x i8> [[X_ROUNDEDUP]] ; %x.lowbits = and <2 x i8> %x, %x.lowbits.are.zero = icmp eq <2 x i8> %x.lowbits, - %x.biased = add <2 x i8> %x, + %x.biased = add <2 x i8> %x, %x.biased.highbits = and <2 x i8> %x.biased, %x.roundedup = select <2 x i1> %x.lowbits.are.zero, <2 x i8> %x, <2 x i8> %x.biased.highbits ret <2 x i8> %x.roundedup } -define <2 x i8> @t8_nonsplat_masked_by_undef_0b0100(<2 x i8> %x) { -; CHECK-LABEL: @t8_nonsplat_masked_by_undef_0b0100( +define <2 x i8> @t8_nonsplat_masked_by_poison_0b0100(<2 x i8> %x) { +; CHECK-LABEL: @t8_nonsplat_masked_by_poison_0b0100( ; CHECK-NEXT: [[X_LOWBITS:%.*]] = and <2 x i8> [[X:%.*]], -; CHECK-NEXT: [[X_LOWBITS_ARE_ZERO:%.*]] = icmp eq <2 x i8> [[X_LOWBITS]], +; CHECK-NEXT: [[X_LOWBITS_ARE_ZERO:%.*]] = icmp eq <2 x i8> [[X_LOWBITS]], ; CHECK-NEXT: [[X_BIASED:%.*]] = add <2 x i8> [[X]], ; CHECK-NEXT: [[X_BIASED_HIGHBITS:%.*]] = and <2 x i8> [[X_BIASED]], ; CHECK-NEXT: [[X_ROUNDEDUP:%.*]] = select <2 x i1> [[X_LOWBITS_ARE_ZERO]], <2 x i8> [[X]], <2 x i8> [[X_BIASED_HIGHBITS]] ; CHECK-NEXT: ret <2 x i8> [[X_ROUNDEDUP]] ; %x.lowbits = and <2 x i8> %x, - %x.lowbits.are.zero = icmp eq <2 x i8> %x.lowbits, + %x.lowbits.are.zero = icmp eq <2 x i8> %x.lowbits, %x.biased = add <2 x i8> %x, %x.biased.highbits = and <2 x i8> %x.biased, %x.roundedup = select <2 x i1> %x.lowbits.are.zero, <2 x i8> %x, <2 x i8> %x.biased.highbits ret <2 x i8> %x.roundedup } -define <2 x i8> @t8_nonsplat_masked_by_undef_0b1000(<2 x i8> %x) { -; CHECK-LABEL: @t8_nonsplat_masked_by_undef_0b1000( -; CHECK-NEXT: [[X_LOWBITS:%.*]] = and <2 x i8> [[X:%.*]], +define <2 x i8> @t8_nonsplat_masked_by_poison_0b1000(<2 x i8> %x) { +; CHECK-LABEL: @t8_nonsplat_masked_by_poison_0b1000( +; CHECK-NEXT: [[X_LOWBITS:%.*]] = and <2 x i8> [[X:%.*]], ; CHECK-NEXT: [[X_LOWBITS_ARE_ZERO:%.*]] = icmp eq <2 x i8> [[X_LOWBITS]], zeroinitializer ; CHECK-NEXT: [[X_BIASED:%.*]] = add <2 x i8> [[X]], ; CHECK-NEXT: [[X_BIASED_HIGHBITS:%.*]] = and <2 x i8> [[X_BIASED]], ; CHECK-NEXT: [[X_ROUNDEDUP:%.*]] = select <2 x i1> [[X_LOWBITS_ARE_ZERO]], <2 x i8> [[X]], <2 x i8> [[X_BIASED_HIGHBITS]] ; CHECK-NEXT: ret <2 x i8> [[X_ROUNDEDUP]] ; - %x.lowbits = and <2 x i8> %x, + %x.lowbits = and <2 x i8> %x, %x.lowbits.are.zero = icmp eq <2 x i8> %x.lowbits, %x.biased = add <2 x i8> %x, %x.biased.highbits = and <2 x i8> %x.biased, @@ -442,28 +442,28 @@ define i8 @t17_oneuse(i8 %x) { define <2 x i4> @t18_replacement_0b0001(<2 x i4> %x) { ; CHECK-LABEL: @t18_replacement_0b0001( ; CHECK-NEXT: [[X_BIASED:%.*]] = add <2 x i4> [[X:%.*]], -; CHECK-NEXT: [[X_BIASED_HIGHBITS:%.*]] = and <2 x i4> [[X_BIASED]], +; CHECK-NEXT: [[X_BIASED_HIGHBITS:%.*]] = and <2 x i4> [[X_BIASED]], ; CHECK-NEXT: call void @use.v2i4(<2 x i4> [[X_BIASED_HIGHBITS]]) ; CHECK-NEXT: ret <2 x i4> [[X_BIASED_HIGHBITS]] ; %x.lowbits = and <2 x i4> %x, %x.lowbits.are.zero = icmp eq <2 x i4> %x.lowbits, %x.biased = add <2 x i4> %x, - %x.biased.highbits = and <2 x i4> %x.biased, + %x.biased.highbits = and <2 x i4> %x.biased, call void @use.v2i4(<2 x i4> %x.biased.highbits) %x.roundedup = select <2 x i1> %x.lowbits.are.zero, <2 x i4> %x, <2 x i4> %x.biased.highbits ret <2 x i4> %x.roundedup } define <2 x i4> @t18_replacement_0b0010(<2 x i4> %x) { ; CHECK-LABEL: @t18_replacement_0b0010( -; CHECK-NEXT: [[X_BIASED:%.*]] = add <2 x i4> [[X:%.*]], +; CHECK-NEXT: [[X_BIASED:%.*]] = add <2 x i4> [[X:%.*]], ; CHECK-NEXT: [[X_BIASED_HIGHBITS:%.*]] = and <2 x i4> [[X_BIASED]], ; CHECK-NEXT: call void @use.v2i4(<2 x i4> [[X_BIASED_HIGHBITS]]) ; CHECK-NEXT: ret <2 x i4> [[X_BIASED_HIGHBITS]] ; %x.lowbits = and <2 x i4> %x, %x.lowbits.are.zero = icmp eq <2 x i4> %x.lowbits, - %x.biased = add <2 x i4> %x, + %x.biased = add <2 x i4> %x, %x.biased.highbits = and <2 x i4> %x.biased, call void @use.v2i4(<2 x i4> %x.biased.highbits) %x.roundedup = select <2 x i1> %x.lowbits.are.zero, <2 x i4> %x, <2 x i4> %x.biased.highbits @@ -477,7 +477,7 @@ define <2 x i4> @t18_replacement_0b0100(<2 x i4> %x) { ; CHECK-NEXT: ret <2 x i4> [[X_BIASED_HIGHBITS]] ; %x.lowbits = and <2 x i4> %x, - %x.lowbits.are.zero = icmp eq <2 x i4> %x.lowbits, + %x.lowbits.are.zero = icmp eq <2 x i4> %x.lowbits, %x.biased = add <2 x i4> %x, %x.biased.highbits = and <2 x i4> %x.biased, call void @use.v2i4(<2 x i4> %x.biased.highbits) @@ -491,7 +491,7 @@ define <2 x i4> @t18_replacement_0b1000(<2 x i4> %x) { ; CHECK-NEXT: call void @use.v2i4(<2 x i4> [[X_BIASED_HIGHBITS]]) ; CHECK-NEXT: ret <2 x i4> [[X_BIASED_HIGHBITS]] ; - %x.lowbits = and <2 x i4> %x, + %x.lowbits = and <2 x i4> %x, %x.lowbits.are.zero = icmp eq <2 x i4> %x.lowbits, %x.biased = add <2 x i4> %x, %x.biased.highbits = and <2 x i4> %x.biased, diff --git a/llvm/test/Transforms/InstCombine/invert-variable-mask-in-masked-merge-vector.ll b/llvm/test/Transforms/InstCombine/invert-variable-mask-in-masked-merge-vector.ll index 486113202ddd..a76662c4bc43 100644 --- a/llvm/test/Transforms/InstCombine/invert-variable-mask-in-masked-merge-vector.ll +++ b/llvm/test/Transforms/InstCombine/invert-variable-mask-in-masked-merge-vector.ll @@ -20,14 +20,14 @@ define <2 x i4> @vector (<2 x i4> %x, <2 x i4> %y, <2 x i4> %m) { ret <2 x i4> %r } -define <3 x i4> @vector_undef (<3 x i4> %x, <3 x i4> %y, <3 x i4> %m) { -; CHECK-LABEL: @vector_undef( +define <3 x i4> @vector_poison (<3 x i4> %x, <3 x i4> %y, <3 x i4> %m) { +; CHECK-LABEL: @vector_poison( ; CHECK-NEXT: [[N0:%.*]] = xor <3 x i4> [[X:%.*]], [[Y:%.*]] ; CHECK-NEXT: [[TMP1:%.*]] = and <3 x i4> [[N0]], [[M:%.*]] ; CHECK-NEXT: [[R:%.*]] = xor <3 x i4> [[TMP1]], [[X]] ; CHECK-NEXT: ret <3 x i4> [[R]] ; - %im = xor <3 x i4> %m, + %im = xor <3 x i4> %m, %n0 = xor <3 x i4> %x, %y %n1 = and <3 x i4> %n0, %im %r = xor <3 x i4> %n1, %y @@ -78,17 +78,17 @@ define <2 x i4> @in_constant_varx_6_invmask_nonsplat(<2 x i4> %x, <2 x i4> %mask ret <2 x i4> %r } -define <3 x i4> @in_constant_varx_6_invmask_undef(<3 x i4> %x, <3 x i4> %mask) { -; CHECK-LABEL: @in_constant_varx_6_invmask_undef( -; CHECK-NEXT: [[N0:%.*]] = xor <3 x i4> [[X:%.*]], +define <3 x i4> @in_constant_varx_6_invmask_poison(<3 x i4> %x, <3 x i4> %mask) { +; CHECK-LABEL: @in_constant_varx_6_invmask_poison( +; CHECK-NEXT: [[N0:%.*]] = xor <3 x i4> [[X:%.*]], ; CHECK-NEXT: [[TMP1:%.*]] = and <3 x i4> [[N0]], [[MASK:%.*]] ; CHECK-NEXT: [[R:%.*]] = xor <3 x i4> [[TMP1]], [[X]] ; CHECK-NEXT: ret <3 x i4> [[R]] ; - %notmask = xor <3 x i4> %mask, - %n0 = xor <3 x i4> %x, ; %x + %notmask = xor <3 x i4> %mask, + %n0 = xor <3 x i4> %x, ; %x %n1 = and <3 x i4> %n0, %notmask - %r = xor <3 x i4> %n1, + %r = xor <3 x i4> %n1, ret <3 x i4> %r } @@ -133,15 +133,15 @@ define <2 x i4> @in_constant_6_vary_invmask_nonsplat(<2 x i4> %y, <2 x i4> %mask ret <2 x i4> %r } -define <3 x i4> @in_constant_6_vary_invmask_undef(<3 x i4> %y, <3 x i4> %mask) { -; CHECK-LABEL: @in_constant_6_vary_invmask_undef( -; CHECK-NEXT: [[N0:%.*]] = xor <3 x i4> [[Y:%.*]], +define <3 x i4> @in_constant_6_vary_invmask_poison(<3 x i4> %y, <3 x i4> %mask) { +; CHECK-LABEL: @in_constant_6_vary_invmask_poison( +; CHECK-NEXT: [[N0:%.*]] = xor <3 x i4> [[Y:%.*]], ; CHECK-NEXT: [[TMP1:%.*]] = and <3 x i4> [[N0]], [[MASK:%.*]] -; CHECK-NEXT: [[R:%.*]] = xor <3 x i4> [[TMP1]], +; CHECK-NEXT: [[R:%.*]] = xor <3 x i4> [[TMP1]], ; CHECK-NEXT: ret <3 x i4> [[R]] ; - %notmask = xor <3 x i4> %mask, - %n0 = xor <3 x i4> %y, ; %x + %notmask = xor <3 x i4> %mask, + %n0 = xor <3 x i4> %y, ; %x %n1 = and <3 x i4> %n0, %notmask %r = xor <3 x i4> %n1, %y ret <3 x i4> %r diff --git a/llvm/test/Transforms/InstCombine/lshr-and-negC-icmpeq-zero.ll b/llvm/test/Transforms/InstCombine/lshr-and-negC-icmpeq-zero.ll index 847a7940bad8..5d058b20be72 100644 --- a/llvm/test/Transforms/InstCombine/lshr-and-negC-icmpeq-zero.ll +++ b/llvm/test/Transforms/InstCombine/lshr-and-negC-icmpeq-zero.ll @@ -81,39 +81,39 @@ define <4 x i1> @vec_4xi32_lshr_and_negC_eq(<4 x i32> %x, <4 x i32> %y) { ret <4 x i1> %r } -define <4 x i1> @vec_lshr_and_negC_eq_undef1(<4 x i32> %x, <4 x i32> %y) { -; CHECK-LABEL: @vec_lshr_and_negC_eq_undef1( +define <4 x i1> @vec_lshr_and_negC_eq_poison1(<4 x i32> %x, <4 x i32> %y) { +; CHECK-LABEL: @vec_lshr_and_negC_eq_poison1( ; CHECK-NEXT: [[LSHR:%.*]] = lshr <4 x i32> [[X:%.*]], [[Y:%.*]] ; CHECK-NEXT: [[R:%.*]] = icmp ult <4 x i32> [[LSHR]], ; CHECK-NEXT: ret <4 x i1> [[R]] ; %lshr = lshr <4 x i32> %x, %y - %and = and <4 x i32> %lshr, ; ~7 + %and = and <4 x i32> %lshr, ; ~7 %r = icmp eq <4 x i32> %and, ret <4 x i1> %r } -define <4 x i1> @vec_lshr_and_negC_eq_undef2(<4 x i32> %x, <4 x i32> %y) { -; CHECK-LABEL: @vec_lshr_and_negC_eq_undef2( +define <4 x i1> @vec_lshr_and_negC_eq_poison2(<4 x i32> %x, <4 x i32> %y) { +; CHECK-LABEL: @vec_lshr_and_negC_eq_poison2( ; CHECK-NEXT: [[LSHR:%.*]] = lshr <4 x i32> [[X:%.*]], [[Y:%.*]] ; CHECK-NEXT: [[R:%.*]] = icmp ult <4 x i32> [[LSHR]], ; CHECK-NEXT: ret <4 x i1> [[R]] ; %lshr = lshr <4 x i32> %x, %y %and = and <4 x i32> %lshr, ; ~7 - %r = icmp eq <4 x i32> %and, + %r = icmp eq <4 x i32> %and, ret <4 x i1> %r } -define <4 x i1> @vec_lshr_and_negC_eq_undef3(<4 x i32> %x, <4 x i32> %y) { -; CHECK-LABEL: @vec_lshr_and_negC_eq_undef3( +define <4 x i1> @vec_lshr_and_negC_eq_poison3(<4 x i32> %x, <4 x i32> %y) { +; CHECK-LABEL: @vec_lshr_and_negC_eq_poison3( ; CHECK-NEXT: [[LSHR:%.*]] = lshr <4 x i32> [[X:%.*]], [[Y:%.*]] ; CHECK-NEXT: [[R:%.*]] = icmp ult <4 x i32> [[LSHR]], ; CHECK-NEXT: ret <4 x i1> [[R]] ; %lshr = lshr <4 x i32> %x, %y - %and = and <4 x i32> %lshr, ; ~7 - %r = icmp eq <4 x i32> %and, + %and = and <4 x i32> %lshr, ; ~7 + %r = icmp eq <4 x i32> %and, ret <4 x i1> %r } diff --git a/llvm/test/Transforms/InstCombine/lshr-and-signbit-icmpeq-zero.ll b/llvm/test/Transforms/InstCombine/lshr-and-signbit-icmpeq-zero.ll index 39f4e58b25dc..0166680309ea 100644 --- a/llvm/test/Transforms/InstCombine/lshr-and-signbit-icmpeq-zero.ll +++ b/llvm/test/Transforms/InstCombine/lshr-and-signbit-icmpeq-zero.ll @@ -81,39 +81,39 @@ define <4 x i1> @vec_4xi32_lshr_and_signbit_eq(<4 x i32> %x, <4 x i32> %y) { ret <4 x i1> %r } -define <4 x i1> @vec_4xi32_lshr_and_signbit_eq_undef1(<4 x i32> %x, <4 x i32> %y) { -; CHECK-LABEL: @vec_4xi32_lshr_and_signbit_eq_undef1( +define <4 x i1> @vec_4xi32_lshr_and_signbit_eq_poison1(<4 x i32> %x, <4 x i32> %y) { +; CHECK-LABEL: @vec_4xi32_lshr_and_signbit_eq_poison1( ; CHECK-NEXT: [[LSHR:%.*]] = lshr <4 x i32> [[X:%.*]], [[Y:%.*]] ; CHECK-NEXT: [[R:%.*]] = icmp sgt <4 x i32> [[LSHR]], ; CHECK-NEXT: ret <4 x i1> [[R]] ; %lshr = lshr <4 x i32> %x, %y - %and = and <4 x i32> %lshr, + %and = and <4 x i32> %lshr, %r = icmp eq <4 x i32> %and, ret <4 x i1> %r } -define <4 x i1> @vec_4xi32_lshr_and_signbit_eq_undef2(<4 x i32> %x, <4 x i32> %y) { -; CHECK-LABEL: @vec_4xi32_lshr_and_signbit_eq_undef2( +define <4 x i1> @vec_4xi32_lshr_and_signbit_eq_poison2(<4 x i32> %x, <4 x i32> %y) { +; CHECK-LABEL: @vec_4xi32_lshr_and_signbit_eq_poison2( ; CHECK-NEXT: [[LSHR:%.*]] = lshr <4 x i32> [[X:%.*]], [[Y:%.*]] ; CHECK-NEXT: [[R:%.*]] = icmp sgt <4 x i32> [[LSHR]], ; CHECK-NEXT: ret <4 x i1> [[R]] ; %lshr = lshr <4 x i32> %x, %y %and = and <4 x i32> %lshr, - %r = icmp eq <4 x i32> %and, + %r = icmp eq <4 x i32> %and, ret <4 x i1> %r } -define <4 x i1> @vec_4xi32_lshr_and_signbit_eq_undef3(<4 x i32> %x, <4 x i32> %y) { -; CHECK-LABEL: @vec_4xi32_lshr_and_signbit_eq_undef3( +define <4 x i1> @vec_4xi32_lshr_and_signbit_eq_poison3(<4 x i32> %x, <4 x i32> %y) { +; CHECK-LABEL: @vec_4xi32_lshr_and_signbit_eq_poison3( ; CHECK-NEXT: [[LSHR:%.*]] = lshr <4 x i32> [[X:%.*]], [[Y:%.*]] ; CHECK-NEXT: [[R:%.*]] = icmp sgt <4 x i32> [[LSHR]], ; CHECK-NEXT: ret <4 x i1> [[R]] ; %lshr = lshr <4 x i32> %x, %y - %and = and <4 x i32> %lshr, - %r = icmp eq <4 x i32> %and, + %and = and <4 x i32> %lshr, + %r = icmp eq <4 x i32> %and, ret <4 x i1> %r } diff --git a/llvm/test/Transforms/InstCombine/masked-merge-add.ll b/llvm/test/Transforms/InstCombine/masked-merge-add.ll index f655153108a4..0484369e99d6 100644 --- a/llvm/test/Transforms/InstCombine/masked-merge-add.ll +++ b/llvm/test/Transforms/InstCombine/masked-merge-add.ll @@ -51,7 +51,7 @@ define <3 x i32> @p_vec_undef(<3 x i32> %x, <3 x i32> %y, <3 x i32> noundef %m) ; CHECK-NEXT: [[AND:%.*]] = and <3 x i32> [[X:%.*]], [[M:%.*]] ; CHECK-NEXT: [[NEG:%.*]] = xor <3 x i32> [[M]], ; CHECK-NEXT: [[AND1:%.*]] = and <3 x i32> [[NEG]], [[Y:%.*]] -; CHECK-NEXT: [[RET:%.*]] = or disjoint <3 x i32> [[AND]], [[AND1]] +; CHECK-NEXT: [[RET:%.*]] = add <3 x i32> [[AND]], [[AND1]] ; CHECK-NEXT: ret <3 x i32> [[RET]] ; %and = and <3 x i32> %x, %m @@ -61,6 +61,21 @@ define <3 x i32> @p_vec_undef(<3 x i32> %x, <3 x i32> %y, <3 x i32> noundef %m) ret <3 x i32> %ret } +define <3 x i32> @p_vec_poison(<3 x i32> %x, <3 x i32> %y, <3 x i32> noundef %m) { +; CHECK-LABEL: @p_vec_poison( +; CHECK-NEXT: [[AND:%.*]] = and <3 x i32> [[X:%.*]], [[M:%.*]] +; CHECK-NEXT: [[NEG:%.*]] = xor <3 x i32> [[M]], +; CHECK-NEXT: [[AND1:%.*]] = and <3 x i32> [[NEG]], [[Y:%.*]] +; CHECK-NEXT: [[RET:%.*]] = or disjoint <3 x i32> [[AND]], [[AND1]] +; CHECK-NEXT: ret <3 x i32> [[RET]] +; + %and = and <3 x i32> %x, %m + %neg = xor <3 x i32> %m, + %and1 = and <3 x i32> %neg, %y + %ret = add <3 x i32> %and, %and1 + ret <3 x i32> %ret +} + ; ============================================================================ ; ; Constant mask. ; ============================================================================ ; diff --git a/llvm/test/Transforms/InstCombine/masked-merge-or.ll b/llvm/test/Transforms/InstCombine/masked-merge-or.ll index b49ec07706e2..0531a532fc7e 100644 --- a/llvm/test/Transforms/InstCombine/masked-merge-or.ll +++ b/llvm/test/Transforms/InstCombine/masked-merge-or.ll @@ -51,7 +51,7 @@ define <3 x i32> @p_vec_undef(<3 x i32> %x, <3 x i32> %y, <3 x i32> noundef %m) ; CHECK-NEXT: [[AND:%.*]] = and <3 x i32> [[X:%.*]], [[M:%.*]] ; CHECK-NEXT: [[NEG:%.*]] = xor <3 x i32> [[M]], ; CHECK-NEXT: [[AND1:%.*]] = and <3 x i32> [[NEG]], [[Y:%.*]] -; CHECK-NEXT: [[RET:%.*]] = or disjoint <3 x i32> [[AND]], [[AND1]] +; CHECK-NEXT: [[RET:%.*]] = or <3 x i32> [[AND]], [[AND1]] ; CHECK-NEXT: ret <3 x i32> [[RET]] ; %and = and <3 x i32> %x, %m @@ -61,6 +61,21 @@ define <3 x i32> @p_vec_undef(<3 x i32> %x, <3 x i32> %y, <3 x i32> noundef %m) ret <3 x i32> %ret } +define <3 x i32> @p_vec_poison(<3 x i32> %x, <3 x i32> %y, <3 x i32> noundef %m) { +; CHECK-LABEL: @p_vec_poison( +; CHECK-NEXT: [[AND:%.*]] = and <3 x i32> [[X:%.*]], [[M:%.*]] +; CHECK-NEXT: [[NEG:%.*]] = xor <3 x i32> [[M]], +; CHECK-NEXT: [[AND1:%.*]] = and <3 x i32> [[NEG]], [[Y:%.*]] +; CHECK-NEXT: [[RET:%.*]] = or disjoint <3 x i32> [[AND]], [[AND1]] +; CHECK-NEXT: ret <3 x i32> [[RET]] +; + %and = and <3 x i32> %x, %m + %neg = xor <3 x i32> %m, + %and1 = and <3 x i32> %neg, %y + %ret = or <3 x i32> %and, %and1 + ret <3 x i32> %ret +} + ; ============================================================================ ; ; Constant mask. ; ============================================================================ ; diff --git a/llvm/test/Transforms/InstCombine/masked-merge-xor.ll b/llvm/test/Transforms/InstCombine/masked-merge-xor.ll index a6d201be68ce..74cc7625aebf 100644 --- a/llvm/test/Transforms/InstCombine/masked-merge-xor.ll +++ b/llvm/test/Transforms/InstCombine/masked-merge-xor.ll @@ -51,7 +51,7 @@ define <3 x i32> @p_vec_undef(<3 x i32> %x, <3 x i32> %y, <3 x i32> noundef %m) ; CHECK-NEXT: [[AND:%.*]] = and <3 x i32> [[X:%.*]], [[M:%.*]] ; CHECK-NEXT: [[NEG:%.*]] = xor <3 x i32> [[M]], ; CHECK-NEXT: [[AND1:%.*]] = and <3 x i32> [[NEG]], [[Y:%.*]] -; CHECK-NEXT: [[RET:%.*]] = or disjoint <3 x i32> [[AND]], [[AND1]] +; CHECK-NEXT: [[RET:%.*]] = xor <3 x i32> [[AND]], [[AND1]] ; CHECK-NEXT: ret <3 x i32> [[RET]] ; %and = and <3 x i32> %x, %m @@ -61,6 +61,21 @@ define <3 x i32> @p_vec_undef(<3 x i32> %x, <3 x i32> %y, <3 x i32> noundef %m) ret <3 x i32> %ret } +define <3 x i32> @p_vec_poison(<3 x i32> %x, <3 x i32> %y, <3 x i32> noundef %m) { +; CHECK-LABEL: @p_vec_poison( +; CHECK-NEXT: [[AND:%.*]] = and <3 x i32> [[X:%.*]], [[M:%.*]] +; CHECK-NEXT: [[NEG:%.*]] = xor <3 x i32> [[M]], +; CHECK-NEXT: [[AND1:%.*]] = and <3 x i32> [[NEG]], [[Y:%.*]] +; CHECK-NEXT: [[RET:%.*]] = or disjoint <3 x i32> [[AND]], [[AND1]] +; CHECK-NEXT: ret <3 x i32> [[RET]] +; + %and = and <3 x i32> %x, %m + %neg = xor <3 x i32> %m, + %and1 = and <3 x i32> %neg, %y + %ret = xor <3 x i32> %and, %and1 + ret <3 x i32> %ret +} + ; ============================================================================ ; ; Constant mask. ; ============================================================================ ; diff --git a/llvm/test/Transforms/InstCombine/min-positive.ll b/llvm/test/Transforms/InstCombine/min-positive.ll index 1fb212b73872..d2c2e9018792 100644 --- a/llvm/test/Transforms/InstCombine/min-positive.ll +++ b/llvm/test/Transforms/InstCombine/min-positive.ll @@ -67,16 +67,16 @@ define <2 x i1> @smin_commute_vec(<2 x i32> %x, <2 x i32> %other) { ret <2 x i1> %test } -define <2 x i1> @smin_commute_vec_undef_elts(<2 x i32> %x, <2 x i32> %other) { -; CHECK-LABEL: @smin_commute_vec_undef_elts( -; CHECK-NEXT: [[TEST:%.*]] = icmp sgt <2 x i32> [[OTHER:%.*]], +define <2 x i1> @smin_commute_vec_poison_elts(<2 x i32> %x, <2 x i32> %other) { +; CHECK-LABEL: @smin_commute_vec_poison_elts( +; CHECK-NEXT: [[TEST:%.*]] = icmp sgt <2 x i32> [[OTHER:%.*]], ; CHECK-NEXT: ret <2 x i1> [[TEST]] ; %notneg = and <2 x i32> %x, %positive = or <2 x i32> %notneg, %cmp = icmp slt <2 x i32> %other, %positive %sel = select <2 x i1> %cmp, <2 x i32> %other, <2 x i32> %positive - %test = icmp sgt <2 x i32> %sel, + %test = icmp sgt <2 x i32> %sel, ret <2 x i1> %test } ; %positive might be zero diff --git a/llvm/test/Transforms/InstCombine/minmax-fold.ll b/llvm/test/Transforms/InstCombine/minmax-fold.ll index bbbbf9eb6eaf..8b47dc7a2807 100644 --- a/llvm/test/Transforms/InstCombine/minmax-fold.ll +++ b/llvm/test/Transforms/InstCombine/minmax-fold.ll @@ -1360,14 +1360,15 @@ define i8 @PR14613_smax(i8 %x) { define i8 @PR46271(<2 x i8> %x) { ; CHECK-LABEL: @PR46271( -; CHECK-NEXT: [[TMP1:%.*]] = call <2 x i8> @llvm.smax.v2i8(<2 x i8> [[X:%.*]], <2 x i8> ) +; CHECK-NEXT: [[TMP3:%.*]] = xor <2 x i8> [[X:%.*]], +; CHECK-NEXT: [[A_INV:%.*]] = icmp slt <2 x i8> [[X]], zeroinitializer +; CHECK-NEXT: [[TMP1:%.*]] = select <2 x i1> [[A_INV]], <2 x i8> , <2 x i8> [[TMP3]] ; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i8> [[TMP1]], i64 1 -; CHECK-NEXT: [[R:%.*]] = xor i8 [[TMP2]], -1 -; CHECK-NEXT: ret i8 [[R]] +; CHECK-NEXT: ret i8 [[TMP2]] ; %a = icmp sgt <2 x i8> %x, - %b = select <2 x i1> %a, <2 x i8> %x, <2 x i8> - %not = xor <2 x i8> %b, + %b = select <2 x i1> %a, <2 x i8> %x, <2 x i8> + %not = xor <2 x i8> %b, %r = extractelement <2 x i8> %not, i32 1 ret i8 %r } diff --git a/llvm/test/Transforms/InstCombine/minmax-intrinsics.ll b/llvm/test/Transforms/InstCombine/minmax-intrinsics.ll index bd1a47bbfcc1..a76f0f84ba34 100644 --- a/llvm/test/Transforms/InstCombine/minmax-intrinsics.ll +++ b/llvm/test/Transforms/InstCombine/minmax-intrinsics.ll @@ -393,7 +393,7 @@ define i8 @smax_of_nots(i8 %x, i8 %y) { ret i8 %m } -; Vectors are ok (including undef lanes of not ops) +; Vectors are ok (including poison lanes of not ops) define <3 x i8> @smin_of_nots(<3 x i8> %x, <3 x i8> %y) { ; CHECK-LABEL: @smin_of_nots( @@ -401,8 +401,8 @@ define <3 x i8> @smin_of_nots(<3 x i8> %x, <3 x i8> %y) { ; CHECK-NEXT: [[M:%.*]] = xor <3 x i8> [[TMP1]], ; CHECK-NEXT: ret <3 x i8> [[M]] ; - %notx = xor <3 x i8> %x, - %noty = xor <3 x i8> %y, + %notx = xor <3 x i8> %x, + %noty = xor <3 x i8> %y, %m = call <3 x i8> @llvm.smin.v3i8(<3 x i8> %notx, <3 x i8> %noty) ret <3 x i8> %m } @@ -473,16 +473,16 @@ define i8 @smax_of_not_and_const(i8 %x) { ret i8 %m } -; Vectors are ok (including undef lanes of not ops and min/max constant operand) +; Vectors are ok (including poison lanes of not ops and min/max constant operand) define <3 x i8> @smin_of_not_and_const(<3 x i8> %x) { ; CHECK-LABEL: @smin_of_not_and_const( -; CHECK-NEXT: [[TMP1:%.*]] = call <3 x i8> @llvm.smax.v3i8(<3 x i8> [[X:%.*]], <3 x i8> ) +; CHECK-NEXT: [[TMP1:%.*]] = call <3 x i8> @llvm.smax.v3i8(<3 x i8> [[X:%.*]], <3 x i8> ) ; CHECK-NEXT: [[M:%.*]] = xor <3 x i8> [[TMP1]], ; CHECK-NEXT: ret <3 x i8> [[M]] ; - %notx = xor <3 x i8> %x, - %m = call <3 x i8> @llvm.smin.v3i8(<3 x i8> , <3 x i8> %notx) + %notx = xor <3 x i8> %x, + %m = call <3 x i8> @llvm.smin.v3i8(<3 x i8> , <3 x i8> %notx) ret <3 x i8> %m } @@ -706,7 +706,7 @@ define <3 x i8> @smax_negation_vec(<3 x i8> %x) { ; CHECK-NEXT: [[R:%.*]] = call <3 x i8> @llvm.abs.v3i8(<3 x i8> [[X:%.*]], i1 false) ; CHECK-NEXT: ret <3 x i8> [[R]] ; - %s = sub <3 x i8> , %x + %s = sub <3 x i8> , %x %r = call <3 x i8> @llvm.smax.v3i8(<3 x i8> %x, <3 x i8> %s) ret <3 x i8> %r } @@ -912,7 +912,7 @@ define <3 x i8> @umin_non_zero_idiom4(<3 x i8> %a) { ; CHECK-NEXT: [[RES:%.*]] = zext <3 x i1> [[TMP1]] to <3 x i8> ; CHECK-NEXT: ret <3 x i8> [[RES]] ; - %res = call <3 x i8> @llvm.umin.v3i8(<3 x i8> %a, <3 x i8> ) + %res = call <3 x i8> @llvm.umin.v3i8(<3 x i8> %a, <3 x i8> ) ret <3 x i8> %res } @@ -2118,15 +2118,15 @@ define i8 @umin_offset_uses(i8 %x) { ret i8 %m } -; TODO: This could transform, but undef element must not propagate to the new add. +; TODO: This could transform -define <3 x i8> @umax_vector_splat_undef(<3 x i8> %x) { -; CHECK-LABEL: @umax_vector_splat_undef( -; CHECK-NEXT: [[A:%.*]] = add nuw <3 x i8> [[X:%.*]], +define <3 x i8> @umax_vector_splat_poison(<3 x i8> %x) { +; CHECK-LABEL: @umax_vector_splat_poison( +; CHECK-NEXT: [[A:%.*]] = add nuw <3 x i8> [[X:%.*]], ; CHECK-NEXT: [[R:%.*]] = call <3 x i8> @llvm.umax.v3i8(<3 x i8> [[A]], <3 x i8> ) ; CHECK-NEXT: ret <3 x i8> [[R]] ; - %a = add nuw <3 x i8> %x, + %a = add nuw <3 x i8> %x, %r = call <3 x i8> @llvm.umax.v3i8(<3 x i8> %a, <3 x i8> ) ret <3 x i8> %r } @@ -2506,8 +2506,8 @@ entry: ret i8 %val } -define <3 x i8> @fold_umax_with_knownbits_info_undef_in_splat(<3 x i8> %a, <3 x i8> %b) { -; CHECK-LABEL: @fold_umax_with_knownbits_info_undef_in_splat( +define <3 x i8> @fold_umax_with_knownbits_info_poison_in_splat(<3 x i8> %a, <3 x i8> %b) { +; CHECK-LABEL: @fold_umax_with_knownbits_info_poison_in_splat( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[A1:%.*]] = or <3 x i8> [[A:%.*]], ; CHECK-NEXT: [[A2:%.*]] = shl <3 x i8> [[B:%.*]], @@ -2518,7 +2518,7 @@ entry: %a1 = or <3 x i8> %a, %a2 = shl <3 x i8> %b, %sub = sub <3 x i8> %a1, %a2 - %val = call <3 x i8> @llvm.umax.v3i8(<3 x i8> %sub, <3 x i8> ) + %val = call <3 x i8> @llvm.umax.v3i8(<3 x i8> %sub, <3 x i8> ) ret <3 x i8> %val } @@ -2535,8 +2535,8 @@ entry: ret i8 %val } -define <3 x i8> @fold_umin_with_knownbits_info_undef_in_splat(<3 x i8> %a, <3 x i8> %b) { -; CHECK-LABEL: @fold_umin_with_knownbits_info_undef_in_splat( +define <3 x i8> @fold_umin_with_knownbits_info_poison_in_splat(<3 x i8> %a, <3 x i8> %b) { +; CHECK-LABEL: @fold_umin_with_knownbits_info_poison_in_splat( ; CHECK-NEXT: entry: ; CHECK-NEXT: ret <3 x i8> ; @@ -2544,7 +2544,7 @@ entry: %a1 = or <3 x i8> %a, %a2 = shl <3 x i8> %b, %sub = sub <3 x i8> %a1, %a2 - %val = call <3 x i8> @llvm.umin.v3i8(<3 x i8> %sub, <3 x i8> ) + %val = call <3 x i8> @llvm.umin.v3i8(<3 x i8> %sub, <3 x i8> ) ret <3 x i8> %val } diff --git a/llvm/test/Transforms/InstCombine/mul-inseltpoison.ll b/llvm/test/Transforms/InstCombine/mul-inseltpoison.ll index 8fe4261bbf00..f47c5577075c 100644 --- a/llvm/test/Transforms/InstCombine/mul-inseltpoison.ll +++ b/llvm/test/Transforms/InstCombine/mul-inseltpoison.ll @@ -784,7 +784,7 @@ define <2 x i8> @negate_if_false_commute(<2 x i8> %px, <2 x i1> %cond) { ; CHECK-NEXT: ret <2 x i8> [[R]] ; %x = sdiv <2 x i8> , %px ; thwart complexity-based canonicalization - %sel = select <2 x i1> %cond, <2 x i8> , <2 x i8> + %sel = select <2 x i1> %cond, <2 x i8> , <2 x i8> %r = mul <2 x i8> %x, %sel ret <2 x i8> %r } @@ -931,7 +931,7 @@ define @mul_scalable_splat_zero( %z) { ; CHECK-LABEL: @mul_scalable_splat_zero( ; CHECK-NEXT: ret zeroinitializer ; - %shuf = shufflevector insertelement ( undef, i64 0, i32 0), poison, zeroinitializer + %shuf = shufflevector insertelement ( poison, i64 0, i32 0), poison, zeroinitializer %t3 = mul %shuf, %z ret %t3 } @@ -973,14 +973,14 @@ define <2 x i32> @mulsub1_vec_nonuniform(<2 x i32> %a0, <2 x i32> %a1) { ret <2 x i32> %mul } -define <2 x i32> @mulsub1_vec_nonuniform_undef(<2 x i32> %a0, <2 x i32> %a1) { -; CHECK-LABEL: @mulsub1_vec_nonuniform_undef( +define <2 x i32> @mulsub1_vec_nonuniform_poison(<2 x i32> %a0, <2 x i32> %a1) { +; CHECK-LABEL: @mulsub1_vec_nonuniform_poison( ; CHECK-NEXT: [[SUB_NEG:%.*]] = sub <2 x i32> [[A0:%.*]], [[A1:%.*]] ; CHECK-NEXT: [[MUL:%.*]] = shl <2 x i32> [[SUB_NEG]], ; CHECK-NEXT: ret <2 x i32> [[MUL]] ; %sub = sub <2 x i32> %a1, %a0 - %mul = mul <2 x i32> %sub, + %mul = mul <2 x i32> %sub, ret <2 x i32> %mul } @@ -1017,14 +1017,14 @@ define <2 x i32> @mulsub2_vec_nonuniform(<2 x i32> %a0) { ret <2 x i32> %mul } -define <2 x i32> @mulsub2_vec_nonuniform_undef(<2 x i32> %a0) { -; CHECK-LABEL: @mulsub2_vec_nonuniform_undef( +define <2 x i32> @mulsub2_vec_nonuniform_poison(<2 x i32> %a0) { +; CHECK-LABEL: @mulsub2_vec_nonuniform_poison( ; CHECK-NEXT: [[SUB_NEG:%.*]] = add <2 x i32> [[A0:%.*]], ; CHECK-NEXT: [[MUL:%.*]] = shl <2 x i32> [[SUB_NEG]], ; CHECK-NEXT: ret <2 x i32> [[MUL]] ; %sub = sub <2 x i32> , %a0 - %mul = mul <2 x i32> %sub, + %mul = mul <2 x i32> %sub, ret <2 x i32> %mul } @@ -1061,14 +1061,14 @@ define <2 x i32> @muladd2_vec_nonuniform(<2 x i32> %a0) { ret <2 x i32> %mul } -define <2 x i32> @muladd2_vec_nonuniform_undef(<2 x i32> %a0) { -; CHECK-LABEL: @muladd2_vec_nonuniform_undef( +define <2 x i32> @muladd2_vec_nonuniform_poison(<2 x i32> %a0) { +; CHECK-LABEL: @muladd2_vec_nonuniform_poison( ; CHECK-NEXT: [[ADD_NEG:%.*]] = sub <2 x i32> , [[A0:%.*]] ; CHECK-NEXT: [[MUL:%.*]] = shl <2 x i32> [[ADD_NEG]], ; CHECK-NEXT: ret <2 x i32> [[MUL]] ; %add = add <2 x i32> %a0, - %mul = mul <2 x i32> %add, + %mul = mul <2 x i32> %add, ret <2 x i32> %mul } diff --git a/llvm/test/Transforms/InstCombine/mul.ll b/llvm/test/Transforms/InstCombine/mul.ll index d4a689c60786..227ca4a6d5cf 100644 --- a/llvm/test/Transforms/InstCombine/mul.ll +++ b/llvm/test/Transforms/InstCombine/mul.ll @@ -1496,7 +1496,7 @@ define <2 x i8> @negate_if_false_commute(<2 x i8> %px, <2 x i1> %cond) { ; CHECK-NEXT: ret <2 x i8> [[R]] ; %x = sdiv <2 x i8> , %px ; thwart complexity-based canonicalization - %sel = select <2 x i1> %cond, <2 x i8> , <2 x i8> + %sel = select <2 x i1> %cond, <2 x i8> , <2 x i8> %r = mul <2 x i8> %x, %sel ret <2 x i8> %r } @@ -1643,7 +1643,7 @@ define @mul_scalable_splat_zero( %z) { ; CHECK-LABEL: @mul_scalable_splat_zero( ; CHECK-NEXT: ret zeroinitializer ; - %shuf = shufflevector insertelement ( undef, i64 0, i32 0), undef, zeroinitializer + %shuf = shufflevector insertelement ( poison, i64 0, i32 0), poison, zeroinitializer %t3 = mul %shuf, %z ret %t3 } @@ -1752,14 +1752,14 @@ define <2 x i32> @mulsub1_vec_nonuniform(<2 x i32> %a0, <2 x i32> %a1) { ret <2 x i32> %mul } -define <2 x i32> @mulsub1_vec_nonuniform_undef(<2 x i32> %a0, <2 x i32> %a1) { -; CHECK-LABEL: @mulsub1_vec_nonuniform_undef( +define <2 x i32> @mulsub1_vec_nonuniform_poison(<2 x i32> %a0, <2 x i32> %a1) { +; CHECK-LABEL: @mulsub1_vec_nonuniform_poison( ; CHECK-NEXT: [[SUB_NEG:%.*]] = sub <2 x i32> [[A0:%.*]], [[A1:%.*]] ; CHECK-NEXT: [[MUL:%.*]] = shl <2 x i32> [[SUB_NEG]], ; CHECK-NEXT: ret <2 x i32> [[MUL]] ; %sub = sub <2 x i32> %a1, %a0 - %mul = mul <2 x i32> %sub, + %mul = mul <2 x i32> %sub, ret <2 x i32> %mul } @@ -1796,14 +1796,14 @@ define <2 x i32> @mulsub2_vec_nonuniform(<2 x i32> %a0) { ret <2 x i32> %mul } -define <2 x i32> @mulsub2_vec_nonuniform_undef(<2 x i32> %a0) { -; CHECK-LABEL: @mulsub2_vec_nonuniform_undef( +define <2 x i32> @mulsub2_vec_nonuniform_poison(<2 x i32> %a0) { +; CHECK-LABEL: @mulsub2_vec_nonuniform_poison( ; CHECK-NEXT: [[SUB_NEG:%.*]] = add <2 x i32> [[A0:%.*]], ; CHECK-NEXT: [[MUL:%.*]] = shl <2 x i32> [[SUB_NEG]], ; CHECK-NEXT: ret <2 x i32> [[MUL]] ; %sub = sub <2 x i32> , %a0 - %mul = mul <2 x i32> %sub, + %mul = mul <2 x i32> %sub, ret <2 x i32> %mul } @@ -1819,15 +1819,15 @@ define i8 @mulsub_nsw(i8 %a1, i8 %a2) { } ; It would be safe to keep the nsw on the shl here, but only because the mul -; to shl transform happens to replace undef with 0. -define <2 x i8> @mulsub_nsw_undef(<2 x i8> %a1, <2 x i8> %a2) { -; CHECK-LABEL: @mulsub_nsw_undef( +; to shl transform happens to replace poison with 0. +define <2 x i8> @mulsub_nsw_poison(<2 x i8> %a1, <2 x i8> %a2) { +; CHECK-LABEL: @mulsub_nsw_poison( ; CHECK-NEXT: [[A_NEG:%.*]] = sub nsw <2 x i8> [[A2:%.*]], [[A1:%.*]] ; CHECK-NEXT: [[MUL:%.*]] = shl <2 x i8> [[A_NEG]], ; CHECK-NEXT: ret <2 x i8> [[MUL]] ; %a = sub nsw <2 x i8> %a1, %a2 - %mul = mul nsw <2 x i8> %a, + %mul = mul nsw <2 x i8> %a, ret <2 x i8> %mul } @@ -1864,14 +1864,14 @@ define <2 x i32> @muladd2_vec_nonuniform(<2 x i32> %a0) { ret <2 x i32> %mul } -define <2 x i32> @muladd2_vec_nonuniform_undef(<2 x i32> %a0) { -; CHECK-LABEL: @muladd2_vec_nonuniform_undef( +define <2 x i32> @muladd2_vec_nonuniform_poison(<2 x i32> %a0) { +; CHECK-LABEL: @muladd2_vec_nonuniform_poison( ; CHECK-NEXT: [[ADD_NEG:%.*]] = sub <2 x i32> , [[A0:%.*]] ; CHECK-NEXT: [[MUL:%.*]] = shl <2 x i32> [[ADD_NEG]], ; CHECK-NEXT: ret <2 x i32> [[MUL]] ; %add = add <2 x i32> %a0, - %mul = mul <2 x i32> %add, + %mul = mul <2 x i32> %add, ret <2 x i32> %mul } diff --git a/llvm/test/Transforms/InstCombine/not-add.ll b/llvm/test/Transforms/InstCombine/not-add.ll index 877f558ffd50..9ba37b6bba39 100644 --- a/llvm/test/Transforms/InstCombine/not-add.ll +++ b/llvm/test/Transforms/InstCombine/not-add.ll @@ -115,26 +115,26 @@ define <4 x i32> @vector_test(<4 x i32> %x, <4 x i32> %y) { ret <4 x i32> %nota } -define <4 x i32> @vector_test_undef(<4 x i32> %x, <4 x i32> %y) { -; CHECK-LABEL: @vector_test_undef( +define <4 x i32> @vector_test_poison(<4 x i32> %x, <4 x i32> %y) { +; CHECK-LABEL: @vector_test_poison( ; CHECK-NEXT: [[NOTA:%.*]] = sub <4 x i32> [[X:%.*]], [[Y:%.*]] ; CHECK-NEXT: ret <4 x i32> [[NOTA]] ; - %notx = xor <4 x i32> %x, + %notx = xor <4 x i32> %x, %a = add <4 x i32> %notx, %y - %nota = xor <4 x i32> %a, + %nota = xor <4 x i32> %a, ret <4 x i32> %nota } -define <4 x i32> @vector_test_undef_nsw_nuw(<4 x i32> %x, <4 x i32> %y) { -; CHECK-LABEL: @vector_test_undef_nsw_nuw( +define <4 x i32> @vector_test_poison_nsw_nuw(<4 x i32> %x, <4 x i32> %y) { +; CHECK-LABEL: @vector_test_poison_nsw_nuw( ; CHECK-NEXT: [[NOTA:%.*]] = sub nuw nsw <4 x i32> [[X:%.*]], [[Y:%.*]] ; CHECK-NEXT: ret <4 x i32> [[NOTA]] ; - %notx = xor <4 x i32> %x, + %notx = xor <4 x i32> %x, %a = add nsw nuw <4 x i32> %notx, %y - %nota = xor <4 x i32> %a, + %nota = xor <4 x i32> %a, ret <4 x i32> %nota } diff --git a/llvm/test/Transforms/InstCombine/not.ll b/llvm/test/Transforms/InstCombine/not.ll index 98b5d9804156..0c2c6195e324 100644 --- a/llvm/test/Transforms/InstCombine/not.ll +++ b/llvm/test/Transforms/InstCombine/not.ll @@ -430,9 +430,9 @@ define <3 x i5> @not_or_neg_commute_vec(<3 x i5> %x, <3 x i5> %p) { ; CHECK-NEXT: ret <3 x i5> [[NOT]] ; %y = mul <3 x i5> %p, ; thwart complexity-based-canonicalization - %s = sub <3 x i5> , %x + %s = sub <3 x i5> , %x %o = or <3 x i5> %y, %s - %not = xor <3 x i5> %o, + %not = xor <3 x i5> %o, ret <3 x i5> %not } diff --git a/llvm/test/Transforms/InstCombine/omit-urem-of-power-of-two-or-zero-when-comparing-with-zero.ll b/llvm/test/Transforms/InstCombine/omit-urem-of-power-of-two-or-zero-when-comparing-with-zero.ll index c16633efe4ce..3fd4a17d972a 100644 --- a/llvm/test/Transforms/InstCombine/omit-urem-of-power-of-two-or-zero-when-comparing-with-zero.ll +++ b/llvm/test/Transforms/InstCombine/omit-urem-of-power-of-two-or-zero-when-comparing-with-zero.ll @@ -95,41 +95,41 @@ define <4 x i1> @p5_vector_urem_by_const__nonsplat(<4 x i32> %x, <4 x i32> %y) { ret <4 x i1> %t2 } -define <4 x i1> @p6_vector_urem_by_const__nonsplat_undef0(<4 x i32> %x, <4 x i32> %y) { -; CHECK-LABEL: @p6_vector_urem_by_const__nonsplat_undef0( -; CHECK-NEXT: [[T0:%.*]] = and <4 x i32> [[X:%.*]], -; CHECK-NEXT: [[T1:%.*]] = urem <4 x i32> [[T0]], -; CHECK-NEXT: [[T2:%.*]] = icmp eq <4 x i32> [[T1]], zeroinitializer +; The poison value in the vector makes the whole function UB. + +define <4 x i1> @p6_vector_urem_by_const__nonsplat_poison0(<4 x i32> %x, <4 x i32> %y) { +; CHECK-LABEL: @p6_vector_urem_by_const__nonsplat_poison0( +; CHECK-NEXT: [[T0:%.*]] = and <4 x i32> [[X:%.*]], +; CHECK-NEXT: [[T2:%.*]] = icmp eq <4 x i32> [[T0]], zeroinitializer ; CHECK-NEXT: ret <4 x i1> [[T2]] ; - %t0 = and <4 x i32> %x, + %t0 = and <4 x i32> %x, %t1 = urem <4 x i32> %t0, ; '6' is clearly not a power of two %t2 = icmp eq <4 x i32> %t1, ret <4 x i1> %t2 } -define <4 x i1> @p7_vector_urem_by_const__nonsplat_undef2(<4 x i32> %x, <4 x i32> %y) { -; CHECK-LABEL: @p7_vector_urem_by_const__nonsplat_undef2( +define <4 x i1> @p7_vector_urem_by_const__nonsplat_poison2(<4 x i32> %x, <4 x i32> %y) { +; CHECK-LABEL: @p7_vector_urem_by_const__nonsplat_poison2( ; CHECK-NEXT: [[T0:%.*]] = and <4 x i32> [[X:%.*]], -; CHECK-NEXT: [[T2:%.*]] = icmp eq <4 x i32> [[T0]], +; CHECK-NEXT: [[T2:%.*]] = icmp eq <4 x i32> [[T0]], ; CHECK-NEXT: ret <4 x i1> [[T2]] ; %t0 = and <4 x i32> %x, ; clearly a power-of-two or zero %t1 = urem <4 x i32> %t0, ; '6' is clearly not a power of two - %t2 = icmp eq <4 x i32> %t1, + %t2 = icmp eq <4 x i32> %t1, ret <4 x i1> %t2 } -define <4 x i1> @p8_vector_urem_by_const__nonsplat_undef3(<4 x i32> %x, <4 x i32> %y) { -; CHECK-LABEL: @p8_vector_urem_by_const__nonsplat_undef3( -; CHECK-NEXT: [[T0:%.*]] = and <4 x i32> [[X:%.*]], -; CHECK-NEXT: [[T1:%.*]] = urem <4 x i32> [[T0]], -; CHECK-NEXT: [[T2:%.*]] = icmp eq <4 x i32> [[T1]], +define <4 x i1> @p8_vector_urem_by_const__nonsplat_poison3(<4 x i32> %x, <4 x i32> %y) { +; CHECK-LABEL: @p8_vector_urem_by_const__nonsplat_poison3( +; CHECK-NEXT: [[T0:%.*]] = and <4 x i32> [[X:%.*]], +; CHECK-NEXT: [[T2:%.*]] = icmp eq <4 x i32> [[T0]], ; CHECK-NEXT: ret <4 x i1> [[T2]] ; - %t0 = and <4 x i32> %x, + %t0 = and <4 x i32> %x, %t1 = urem <4 x i32> %t0, ; '6' is clearly not a power of two - %t2 = icmp eq <4 x i32> %t1, + %t2 = icmp eq <4 x i32> %t1, ret <4 x i1> %t2 } diff --git a/llvm/test/Transforms/InstCombine/operand-complexity.ll b/llvm/test/Transforms/InstCombine/operand-complexity.ll index 62cfc76d9d24..541a15275b61 100644 --- a/llvm/test/Transforms/InstCombine/operand-complexity.ll +++ b/llvm/test/Transforms/InstCombine/operand-complexity.ll @@ -29,15 +29,15 @@ define <2 x i8> @neg_vec(<2 x i8> %x) { ret <2 x i8> %r } -define <2 x i8> @neg_vec_undef(<2 x i8> %x) { -; CHECK-LABEL: @neg_vec_undef( +define <2 x i8> @neg_vec_poison(<2 x i8> %x) { +; CHECK-LABEL: @neg_vec_poison( ; CHECK-NEXT: [[BO:%.*]] = udiv <2 x i8> [[X:%.*]], -; CHECK-NEXT: [[NEGX:%.*]] = sub <2 x i8> , [[X]] +; CHECK-NEXT: [[NEGX:%.*]] = sub <2 x i8> , [[X]] ; CHECK-NEXT: [[R:%.*]] = xor <2 x i8> [[BO]], [[NEGX]] ; CHECK-NEXT: ret <2 x i8> [[R]] ; %bo = udiv <2 x i8> %x, - %negx = sub <2 x i8> , %x + %negx = sub <2 x i8> , %x %r = xor <2 x i8> %negx, %bo ret <2 x i8> %r } @@ -70,15 +70,15 @@ define <2 x i8> @not_vec(<2 x i8> %x) { ret <2 x i8> %r } -define <2 x i8> @not_vec_undef(<2 x i8> %x) { -; CHECK-LABEL: @not_vec_undef( +define <2 x i8> @not_vec_poison(<2 x i8> %x) { +; CHECK-LABEL: @not_vec_poison( ; CHECK-NEXT: [[BO:%.*]] = udiv <2 x i8> [[X:%.*]], -; CHECK-NEXT: [[NOTX:%.*]] = xor <2 x i8> [[X]], +; CHECK-NEXT: [[NOTX:%.*]] = xor <2 x i8> [[X]], ; CHECK-NEXT: [[R:%.*]] = mul <2 x i8> [[BO]], [[NOTX]] ; CHECK-NEXT: ret <2 x i8> [[R]] ; %bo = udiv <2 x i8> %x, - %notx = xor <2 x i8> , %x + %notx = xor <2 x i8> , %x %r = mul <2 x i8> %notx, %bo ret <2 x i8> %r } @@ -134,8 +134,8 @@ define <2 x float> @fneg_vec(<2 x float> %x) { ret <2 x float> %r } -define <2 x float> @fneg_vec_undef(<2 x float> %x) { -; CHECK-LABEL: @fneg_vec_undef( +define <2 x float> @fneg_vec_poison(<2 x float> %x) { +; CHECK-LABEL: @fneg_vec_poison( ; CHECK-NEXT: [[BO:%.*]] = fdiv <2 x float> [[X:%.*]], ; CHECK-NEXT: [[FNEGX:%.*]] = fneg <2 x float> [[X]] ; CHECK-NEXT: [[R:%.*]] = fmul <2 x float> [[BO]], [[FNEGX]] @@ -143,7 +143,7 @@ define <2 x float> @fneg_vec_undef(<2 x float> %x) { ; CHECK-NEXT: ret <2 x float> [[R]] ; %bo = fdiv <2 x float> %x, - %fnegx = fsub <2 x float> , %x + %fnegx = fsub <2 x float> , %x %r = fmul <2 x float> %fnegx, %bo call void @use_vec(<2 x float> %fnegx) ret <2 x float> %r diff --git a/llvm/test/Transforms/InstCombine/or.ll b/llvm/test/Transforms/InstCombine/or.ll index 1b1a6ffbf0f2..6e2085a8bb6c 100644 --- a/llvm/test/Transforms/InstCombine/or.ll +++ b/llvm/test/Transforms/InstCombine/or.ll @@ -262,26 +262,26 @@ define <2 x i1> @and_icmp_eq_0_vector(<2 x i32> %A, <2 x i32> %B) { ret <2 x i1> %D } -define <2 x i1> @and_icmp_eq_0_vector_undef1(<2 x i32> %A, <2 x i32> %B) { -; CHECK-LABEL: @and_icmp_eq_0_vector_undef1( +define <2 x i1> @and_icmp_eq_0_vector_poison1(<2 x i32> %A, <2 x i32> %B) { +; CHECK-LABEL: @and_icmp_eq_0_vector_poison1( ; CHECK-NEXT: [[TMP1:%.*]] = or <2 x i32> [[A:%.*]], [[B:%.*]] ; CHECK-NEXT: [[D:%.*]] = icmp eq <2 x i32> [[TMP1]], zeroinitializer ; CHECK-NEXT: ret <2 x i1> [[D]] ; - %C1 = icmp eq <2 x i32> %A, - %C2 = icmp eq <2 x i32> %B, + %C1 = icmp eq <2 x i32> %A, + %C2 = icmp eq <2 x i32> %B, %D = and <2 x i1> %C1, %C2 ret <2 x i1> %D } -define <2 x i1> @and_icmp_eq_0_vector_undef2(<2 x i32> %A, <2 x i32> %B) { -; CHECK-LABEL: @and_icmp_eq_0_vector_undef2( +define <2 x i1> @and_icmp_eq_0_vector_poison2(<2 x i32> %A, <2 x i32> %B) { +; CHECK-LABEL: @and_icmp_eq_0_vector_poison2( ; CHECK-NEXT: [[TMP1:%.*]] = or <2 x i32> [[A:%.*]], [[B:%.*]] ; CHECK-NEXT: [[D:%.*]] = icmp eq <2 x i32> [[TMP1]], zeroinitializer ; CHECK-NEXT: ret <2 x i1> [[D]] ; - %C1 = icmp eq <2 x i32> %A, - %C2 = icmp eq <2 x i32> %B, + %C1 = icmp eq <2 x i32> %A, + %C2 = icmp eq <2 x i32> %B, %D = and <2 x i1> %C1, %C2 ret <2 x i1> %D } @@ -566,17 +566,17 @@ define <2 x i1> @test37_uniform(<2 x i32> %x) { ret <2 x i1> %ret1 } -define <2 x i1> @test37_undef(<2 x i32> %x) { -; CHECK-LABEL: @test37_undef( -; CHECK-NEXT: [[ADD1:%.*]] = add <2 x i32> [[X:%.*]], -; CHECK-NEXT: [[CMP1:%.*]] = icmp ult <2 x i32> [[ADD1]], -; CHECK-NEXT: [[CMP2:%.*]] = icmp eq <2 x i32> [[X]], +define <2 x i1> @test37_poison(<2 x i32> %x) { +; CHECK-LABEL: @test37_poison( +; CHECK-NEXT: [[ADD1:%.*]] = add <2 x i32> [[X:%.*]], +; CHECK-NEXT: [[CMP1:%.*]] = icmp ult <2 x i32> [[ADD1]], +; CHECK-NEXT: [[CMP2:%.*]] = icmp eq <2 x i32> [[X]], ; CHECK-NEXT: [[RET1:%.*]] = or <2 x i1> [[CMP1]], [[CMP2]] ; CHECK-NEXT: ret <2 x i1> [[RET1]] ; - %add1 = add <2 x i32> %x, - %cmp1 = icmp ult <2 x i32> %add1, - %cmp2 = icmp eq <2 x i32> %x, + %add1 = add <2 x i32> %x, + %cmp1 = icmp ult <2 x i32> %add1, + %cmp2 = icmp eq <2 x i32> %x, %ret1 = or <2 x i1> %cmp1, %cmp2 ret <2 x i1> %ret1 } @@ -874,19 +874,19 @@ define <2 x i1> @test46_uniform(<2 x i8> %c) { ret <2 x i1> %or } -define <2 x i1> @test46_undef(<2 x i8> %c) { -; CHECK-LABEL: @test46_undef( -; CHECK-NEXT: [[C_OFF:%.*]] = add <2 x i8> [[C:%.*]], -; CHECK-NEXT: [[CMP1:%.*]] = icmp ult <2 x i8> [[C_OFF]], -; CHECK-NEXT: [[C_OFF17:%.*]] = add <2 x i8> [[C]], -; CHECK-NEXT: [[CMP2:%.*]] = icmp ult <2 x i8> [[C_OFF17]], +define <2 x i1> @test46_poison(<2 x i8> %c) { +; CHECK-LABEL: @test46_poison( +; CHECK-NEXT: [[C_OFF:%.*]] = add <2 x i8> [[C:%.*]], +; CHECK-NEXT: [[CMP1:%.*]] = icmp ult <2 x i8> [[C_OFF]], +; CHECK-NEXT: [[C_OFF17:%.*]] = add <2 x i8> [[C]], +; CHECK-NEXT: [[CMP2:%.*]] = icmp ult <2 x i8> [[C_OFF17]], ; CHECK-NEXT: [[OR:%.*]] = or <2 x i1> [[CMP1]], [[CMP2]] ; CHECK-NEXT: ret <2 x i1> [[OR]] ; - %c.off = add <2 x i8> %c, - %cmp1 = icmp ult <2 x i8> %c.off, - %c.off17 = add <2 x i8> %c, - %cmp2 = icmp ult <2 x i8> %c.off17, + %c.off = add <2 x i8> %c, + %cmp1 = icmp ult <2 x i8> %c.off, + %c.off17 = add <2 x i8> %c, + %cmp2 = icmp ult <2 x i8> %c.off17, %or = or <2 x i1> %cmp1, %cmp2 ret <2 x i1> %or } diff --git a/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-after-truncation-variant-b.ll b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-after-truncation-variant-b.ll index f0c2f129e3df..5ed7d641df65 100644 --- a/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-after-truncation-variant-b.ll +++ b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-after-truncation-variant-b.ll @@ -89,13 +89,13 @@ define <8 x i32> @t1_vec_splat(<8 x i64> %x, <8 x i32> %nbits) { ret <8 x i32> %t7 } -define <8 x i32> @t2_vec_splat_undef(<8 x i64> %x, <8 x i32> %nbits) { -; CHECK-LABEL: @t2_vec_splat_undef( -; CHECK-NEXT: [[T0:%.*]] = add <8 x i32> [[NBITS:%.*]], +define <8 x i32> @t2_vec_splat_poison(<8 x i64> %x, <8 x i32> %nbits) { +; CHECK-LABEL: @t2_vec_splat_poison( +; CHECK-NEXT: [[T0:%.*]] = add <8 x i32> [[NBITS:%.*]], ; CHECK-NEXT: [[T1:%.*]] = zext <8 x i32> [[T0]] to <8 x i64> -; CHECK-NEXT: [[T2:%.*]] = shl <8 x i64> , [[T1]] -; CHECK-NEXT: [[T3:%.*]] = xor <8 x i64> [[T2]], -; CHECK-NEXT: [[T4:%.*]] = sub <8 x i32> , [[NBITS]] +; CHECK-NEXT: [[T2:%.*]] = shl nsw <8 x i64> , [[T1]] +; CHECK-NEXT: [[T3:%.*]] = xor <8 x i64> [[T2]], +; CHECK-NEXT: [[T4:%.*]] = sub <8 x i32> , [[NBITS]] ; CHECK-NEXT: call void @use8xi32(<8 x i32> [[T0]]) ; CHECK-NEXT: call void @use8xi64(<8 x i64> [[T1]]) ; CHECK-NEXT: call void @use8xi64(<8 x i64> [[T2]]) @@ -106,11 +106,11 @@ define <8 x i32> @t2_vec_splat_undef(<8 x i64> %x, <8 x i32> %nbits) { ; CHECK-NEXT: [[T7:%.*]] = and <8 x i32> [[TMP2]], ; CHECK-NEXT: ret <8 x i32> [[T7]] ; - %t0 = add <8 x i32> %nbits, + %t0 = add <8 x i32> %nbits, %t1 = zext <8 x i32> %t0 to <8 x i64> - %t2 = shl <8 x i64> , %t1 ; shifting by nbits-1 - %t3 = xor <8 x i64> %t2, - %t4 = sub <8 x i32> , %nbits + %t2 = shl <8 x i64> , %t1 ; shifting by nbits-1 + %t3 = xor <8 x i64> %t2, + %t4 = sub <8 x i32> , %nbits call void @use8xi32(<8 x i32> %t0) call void @use8xi64(<8 x i64> %t1) diff --git a/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-after-truncation-variant-c.ll b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-after-truncation-variant-c.ll index 46d1de5781b7..1a711e58c333 100644 --- a/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-after-truncation-variant-c.ll +++ b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-after-truncation-variant-c.ll @@ -73,11 +73,11 @@ define <8 x i32> @t1_vec_splat(<8 x i64> %x, <8 x i32> %nbits) { ret <8 x i32> %t5 } -define <8 x i32> @t2_vec_splat_undef(<8 x i64> %x, <8 x i32> %nbits) { -; CHECK-LABEL: @t2_vec_splat_undef( +define <8 x i32> @t2_vec_splat_poison(<8 x i64> %x, <8 x i32> %nbits) { +; CHECK-LABEL: @t2_vec_splat_poison( ; CHECK-NEXT: [[T0:%.*]] = zext <8 x i32> [[NBITS:%.*]] to <8 x i64> -; CHECK-NEXT: [[T1:%.*]] = lshr <8 x i64> , [[T0]] -; CHECK-NEXT: [[T2:%.*]] = add <8 x i32> [[NBITS]], +; CHECK-NEXT: [[T1:%.*]] = lshr <8 x i64> , [[T0]] +; CHECK-NEXT: [[T2:%.*]] = add <8 x i32> [[NBITS]], ; CHECK-NEXT: call void @use8xi64(<8 x i64> [[T0]]) ; CHECK-NEXT: call void @use8xi64(<8 x i64> [[T1]]) ; CHECK-NEXT: call void @use8xi32(<8 x i32> [[T2]]) @@ -87,8 +87,8 @@ define <8 x i32> @t2_vec_splat_undef(<8 x i64> %x, <8 x i32> %nbits) { ; CHECK-NEXT: ret <8 x i32> [[T5]] ; %t0 = zext <8 x i32> %nbits to <8 x i64> - %t1 = lshr <8 x i64> , %t0 - %t2 = add <8 x i32> %nbits, + %t1 = lshr <8 x i64> , %t0 + %t2 = add <8 x i32> %nbits, call void @use8xi64(<8 x i64> %t0) call void @use8xi64(<8 x i64> %t1) @@ -103,8 +103,8 @@ define <8 x i32> @t2_vec_splat_undef(<8 x i64> %x, <8 x i32> %nbits) { define <8 x i32> @t3_vec_nonsplat(<8 x i64> %x, <8 x i32> %nbits) { ; CHECK-LABEL: @t3_vec_nonsplat( ; CHECK-NEXT: [[T0:%.*]] = zext <8 x i32> [[NBITS:%.*]] to <8 x i64> -; CHECK-NEXT: [[T1:%.*]] = lshr <8 x i64> , [[T0]] -; CHECK-NEXT: [[T2:%.*]] = add <8 x i32> [[NBITS]], +; CHECK-NEXT: [[T1:%.*]] = lshr <8 x i64> , [[T0]] +; CHECK-NEXT: [[T2:%.*]] = add <8 x i32> [[NBITS]], ; CHECK-NEXT: call void @use8xi64(<8 x i64> [[T0]]) ; CHECK-NEXT: call void @use8xi64(<8 x i64> [[T1]]) ; CHECK-NEXT: call void @use8xi32(<8 x i32> [[T2]]) @@ -114,8 +114,8 @@ define <8 x i32> @t3_vec_nonsplat(<8 x i64> %x, <8 x i32> %nbits) { ; CHECK-NEXT: ret <8 x i32> [[T5]] ; %t0 = zext <8 x i32> %nbits to <8 x i64> - %t1 = lshr <8 x i64> , %t0 - %t2 = add <8 x i32> %nbits, + %t1 = lshr <8 x i64> , %t0 + %t2 = add <8 x i32> %nbits, call void @use8xi64(<8 x i64> %t0) call void @use8xi64(<8 x i64> %t1) diff --git a/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-after-truncation-variant-d.ll b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-after-truncation-variant-d.ll index 48873852cfc7..cd0098ecdb0a 100644 --- a/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-after-truncation-variant-d.ll +++ b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-after-truncation-variant-d.ll @@ -81,12 +81,12 @@ define <8 x i32> @t1_vec_splat(<8 x i64> %x, <8 x i32> %nbits) { ret <8 x i32> %t6 } -define <8 x i32> @t2_vec_splat_undef(<8 x i64> %x, <8 x i32> %nbits) { -; CHECK-LABEL: @t2_vec_splat_undef( +define <8 x i32> @t2_vec_splat_poison(<8 x i64> %x, <8 x i32> %nbits) { +; CHECK-LABEL: @t2_vec_splat_poison( ; CHECK-NEXT: [[T0:%.*]] = zext <8 x i32> [[NBITS:%.*]] to <8 x i64> -; CHECK-NEXT: [[T1:%.*]] = shl <8 x i64> , [[T0]] +; CHECK-NEXT: [[T1:%.*]] = shl nsw <8 x i64> , [[T0]] ; CHECK-NEXT: [[T2:%.*]] = lshr <8 x i64> , [[T0]] -; CHECK-NEXT: [[T3:%.*]] = add <8 x i32> [[NBITS]], +; CHECK-NEXT: [[T3:%.*]] = add <8 x i32> [[NBITS]], ; CHECK-NEXT: call void @use8xi64(<8 x i64> [[T0]]) ; CHECK-NEXT: call void @use8xi64(<8 x i64> [[T1]]) ; CHECK-NEXT: call void @use8xi64(<8 x i64> [[T2]]) @@ -97,9 +97,9 @@ define <8 x i32> @t2_vec_splat_undef(<8 x i64> %x, <8 x i32> %nbits) { ; CHECK-NEXT: ret <8 x i32> [[T6]] ; %t0 = zext <8 x i32> %nbits to <8 x i64> - %t1 = shl <8 x i64> , %t0 + %t1 = shl <8 x i64> , %t0 %t2 = lshr <8 x i64> %t1, %t0 - %t3 = add <8 x i32> %nbits, + %t3 = add <8 x i32> %nbits, call void @use8xi64(<8 x i64> %t0) call void @use8xi64(<8 x i64> %t1) @@ -115,9 +115,9 @@ define <8 x i32> @t2_vec_splat_undef(<8 x i64> %x, <8 x i32> %nbits) { define <8 x i32> @t3_vec_nonsplat(<8 x i64> %x, <8 x i32> %nbits) { ; CHECK-LABEL: @t3_vec_nonsplat( ; CHECK-NEXT: [[T0:%.*]] = zext <8 x i32> [[NBITS:%.*]] to <8 x i64> -; CHECK-NEXT: [[T1:%.*]] = shl <8 x i64> , [[T0]] +; CHECK-NEXT: [[T1:%.*]] = shl nsw <8 x i64> , [[T0]] ; CHECK-NEXT: [[T2:%.*]] = lshr <8 x i64> , [[T0]] -; CHECK-NEXT: [[T3:%.*]] = add <8 x i32> [[NBITS]], +; CHECK-NEXT: [[T3:%.*]] = add <8 x i32> [[NBITS]], ; CHECK-NEXT: call void @use8xi64(<8 x i64> [[T0]]) ; CHECK-NEXT: call void @use8xi64(<8 x i64> [[T1]]) ; CHECK-NEXT: call void @use8xi64(<8 x i64> [[T2]]) @@ -128,9 +128,9 @@ define <8 x i32> @t3_vec_nonsplat(<8 x i64> %x, <8 x i32> %nbits) { ; CHECK-NEXT: ret <8 x i32> [[T6]] ; %t0 = zext <8 x i32> %nbits to <8 x i64> - %t1 = shl <8 x i64> , %t0 + %t1 = shl <8 x i64> , %t0 %t2 = lshr <8 x i64> %t1, %t0 - %t3 = add <8 x i32> %nbits, + %t3 = add <8 x i32> %nbits, call void @use8xi64(<8 x i64> %t0) call void @use8xi64(<8 x i64> %t1) diff --git a/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-b.ll b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-b.ll index 8b3f01bcb769..1debf111b18c 100644 --- a/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-b.ll +++ b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-b.ll @@ -71,12 +71,12 @@ define <8 x i32> @t1_vec_splat(<8 x i32> %x, <8 x i32> %nbits) { ret <8 x i32> %t5 } -define <8 x i32> @t1_vec_splat_undef(<8 x i32> %x, <8 x i32> %nbits) { -; CHECK-LABEL: @t1_vec_splat_undef( -; CHECK-NEXT: [[T0:%.*]] = add <8 x i32> [[NBITS:%.*]], -; CHECK-NEXT: [[T1:%.*]] = shl <8 x i32> , [[T0]] -; CHECK-NEXT: [[T2:%.*]] = xor <8 x i32> [[T1]], -; CHECK-NEXT: [[T4:%.*]] = sub <8 x i32> , [[NBITS]] +define <8 x i32> @t1_vec_splat_poison(<8 x i32> %x, <8 x i32> %nbits) { +; CHECK-LABEL: @t1_vec_splat_poison( +; CHECK-NEXT: [[T0:%.*]] = add <8 x i32> [[NBITS:%.*]], +; CHECK-NEXT: [[T1:%.*]] = shl nsw <8 x i32> , [[T0]] +; CHECK-NEXT: [[T2:%.*]] = xor <8 x i32> [[T1]], +; CHECK-NEXT: [[T4:%.*]] = sub <8 x i32> , [[NBITS]] ; CHECK-NEXT: call void @use8xi32(<8 x i32> [[T0]]) ; CHECK-NEXT: call void @use8xi32(<8 x i32> [[T1]]) ; CHECK-NEXT: call void @use8xi32(<8 x i32> [[T2]]) @@ -85,11 +85,11 @@ define <8 x i32> @t1_vec_splat_undef(<8 x i32> %x, <8 x i32> %nbits) { ; CHECK-NEXT: [[T5:%.*]] = and <8 x i32> [[TMP1]], ; CHECK-NEXT: ret <8 x i32> [[T5]] ; - %t0 = add <8 x i32> %nbits, - %t1 = shl <8 x i32> , %t0 - %t2 = xor <8 x i32> %t1, + %t0 = add <8 x i32> %nbits, + %t1 = shl <8 x i32> , %t0 + %t2 = xor <8 x i32> %t1, %t3 = and <8 x i32> %t2, %x - %t4 = sub <8 x i32> , %nbits + %t4 = sub <8 x i32> , %nbits call void @use8xi32(<8 x i32> %t0) call void @use8xi32(<8 x i32> %t1) call void @use8xi32(<8 x i32> %t2) diff --git a/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-c.ll b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-c.ll index 58a905063fac..55d0b3f80a51 100644 --- a/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-c.ll +++ b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-c.ll @@ -55,19 +55,19 @@ define <8 x i32> @t1_vec_splat(<8 x i32> %x, <8 x i32> %nbits) { ret <8 x i32> %t3 } -define <8 x i32> @t1_vec_splat_undef(<8 x i32> %x, <8 x i32> %nbits) { -; CHECK-LABEL: @t1_vec_splat_undef( -; CHECK-NEXT: [[T0:%.*]] = lshr <8 x i32> , [[NBITS:%.*]] -; CHECK-NEXT: [[T2:%.*]] = add <8 x i32> [[NBITS]], +define <8 x i32> @t1_vec_splat_poison(<8 x i32> %x, <8 x i32> %nbits) { +; CHECK-LABEL: @t1_vec_splat_poison( +; CHECK-NEXT: [[T0:%.*]] = lshr <8 x i32> , [[NBITS:%.*]] +; CHECK-NEXT: [[T2:%.*]] = add <8 x i32> [[NBITS]], ; CHECK-NEXT: call void @use8xi32(<8 x i32> [[T0]]) ; CHECK-NEXT: call void @use8xi32(<8 x i32> [[T2]]) ; CHECK-NEXT: [[TMP1:%.*]] = shl <8 x i32> [[X:%.*]], [[T2]] ; CHECK-NEXT: [[T3:%.*]] = and <8 x i32> [[TMP1]], ; CHECK-NEXT: ret <8 x i32> [[T3]] ; - %t0 = lshr <8 x i32> , %nbits + %t0 = lshr <8 x i32> , %nbits %t1 = and <8 x i32> %t0, %x - %t2 = add <8 x i32> %nbits, + %t2 = add <8 x i32> %nbits, call void @use8xi32(<8 x i32> %t0) call void @use8xi32(<8 x i32> %t2) %t3 = shl <8 x i32> %t1, %t2 ; shift is smaller than mask diff --git a/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-d.ll b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-d.ll index 9c096d1418a5..7ad99a6bb0a3 100644 --- a/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-d.ll +++ b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-d.ll @@ -63,11 +63,11 @@ define <8 x i32> @t2_vec_splat(<8 x i32> %x, <8 x i32> %nbits) { ret <8 x i32> %t4 } -define <8 x i32> @t2_vec_splat_undef(<8 x i32> %x, <8 x i32> %nbits) { -; CHECK-LABEL: @t2_vec_splat_undef( -; CHECK-NEXT: [[T0:%.*]] = shl <8 x i32> , [[NBITS:%.*]] +define <8 x i32> @t2_vec_splat_poison(<8 x i32> %x, <8 x i32> %nbits) { +; CHECK-LABEL: @t2_vec_splat_poison( +; CHECK-NEXT: [[T0:%.*]] = shl nsw <8 x i32> , [[NBITS:%.*]] ; CHECK-NEXT: [[T1:%.*]] = lshr <8 x i32> , [[NBITS]] -; CHECK-NEXT: [[T3:%.*]] = add <8 x i32> [[NBITS]], +; CHECK-NEXT: [[T3:%.*]] = add <8 x i32> [[NBITS]], ; CHECK-NEXT: call void @use8xi32(<8 x i32> [[T0]]) ; CHECK-NEXT: call void @use8xi32(<8 x i32> [[T1]]) ; CHECK-NEXT: call void @use8xi32(<8 x i32> [[T3]]) @@ -75,10 +75,10 @@ define <8 x i32> @t2_vec_splat_undef(<8 x i32> %x, <8 x i32> %nbits) { ; CHECK-NEXT: [[T4:%.*]] = and <8 x i32> [[TMP1]], ; CHECK-NEXT: ret <8 x i32> [[T4]] ; - %t0 = shl <8 x i32> , %nbits + %t0 = shl <8 x i32> , %nbits %t1 = lshr <8 x i32> %t0, %nbits %t2 = and <8 x i32> %t1, %x - %t3 = add <8 x i32> %nbits, + %t3 = add <8 x i32> %nbits, call void @use8xi32(<8 x i32> %t0) call void @use8xi32(<8 x i32> %t1) call void @use8xi32(<8 x i32> %t3) diff --git a/llvm/test/Transforms/InstCombine/pr53357.ll b/llvm/test/Transforms/InstCombine/pr53357.ll index 0a6d2993ce46..0ae690869c1c 100644 --- a/llvm/test/Transforms/InstCombine/pr53357.ll +++ b/llvm/test/Transforms/InstCombine/pr53357.ll @@ -30,16 +30,16 @@ define <2 x i32> @src_vec(<2 x i32> noundef %0, <2 x i32> noundef %1) { ret <2 x i32> %6 } -; vector version of src with undef values -define <2 x i32> @src_vec_undef(<2 x i32> noundef %0, <2 x i32> noundef %1) { -; CHECK-LABEL: @src_vec_undef( +; vector version of src with poison values +define <2 x i32> @src_vec_poison(<2 x i32> noundef %0, <2 x i32> noundef %1) { +; CHECK-LABEL: @src_vec_poison( ; CHECK-NEXT: [[TMP3:%.*]] = xor <2 x i32> [[TMP1:%.*]], [[TMP0:%.*]] ; CHECK-NEXT: [[TMP4:%.*]] = xor <2 x i32> [[TMP3]], ; CHECK-NEXT: ret <2 x i32> [[TMP4]] ; %3 = and <2 x i32> %1, %0 %4 = or <2 x i32> %1, %0 - %5 = xor <2 x i32> %4, + %5 = xor <2 x i32> %4, %6 = add <2 x i32> %3, %5 ret <2 x i32> %6 } diff --git a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-b.ll b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-b.ll index d49cfe990d82..cb6775e689b8 100644 --- a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-b.ll +++ b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-b.ll @@ -89,12 +89,12 @@ define <8 x i32> @t1_vec_splat(<8 x i64> %x, <8 x i32> %nbits) { ret <8 x i32> %t6 } -define <8 x i32> @t2_vec_splat_undef(<8 x i64> %x, <8 x i32> %nbits) { -; CHECK-LABEL: @t2_vec_splat_undef( +define <8 x i32> @t2_vec_splat_poison(<8 x i64> %x, <8 x i32> %nbits) { +; CHECK-LABEL: @t2_vec_splat_poison( ; CHECK-NEXT: [[T0:%.*]] = zext <8 x i32> [[NBITS:%.*]] to <8 x i64> -; CHECK-NEXT: [[T1:%.*]] = shl <8 x i64> , [[T0]] -; CHECK-NEXT: [[T2:%.*]] = xor <8 x i64> [[T1]], -; CHECK-NEXT: [[T3:%.*]] = sub <8 x i32> , [[NBITS]] +; CHECK-NEXT: [[T1:%.*]] = shl nsw <8 x i64> , [[T0]] +; CHECK-NEXT: [[T2:%.*]] = xor <8 x i64> [[T1]], +; CHECK-NEXT: [[T3:%.*]] = sub <8 x i32> , [[NBITS]] ; CHECK-NEXT: [[T4:%.*]] = and <8 x i64> [[T2]], [[X:%.*]] ; CHECK-NEXT: call void @use8xi32(<8 x i32> [[NBITS]]) ; CHECK-NEXT: call void @use8xi64(<8 x i64> [[T0]]) @@ -107,9 +107,9 @@ define <8 x i32> @t2_vec_splat_undef(<8 x i64> %x, <8 x i32> %nbits) { ; CHECK-NEXT: ret <8 x i32> [[T6]] ; %t0 = zext <8 x i32> %nbits to <8 x i64> - %t1 = shl <8 x i64> , %t0 - %t2 = xor <8 x i64> %t1, - %t3 = sub <8 x i32> , %nbits + %t1 = shl <8 x i64> , %t0 + %t2 = xor <8 x i64> %t1, + %t3 = sub <8 x i32> , %nbits %t4 = and <8 x i64> %t2, %x call void @use8xi32(<8 x i32> %nbits) diff --git a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-c.ll b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-c.ll index fbbeffbba630..a78246781c7f 100644 --- a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-c.ll +++ b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-c.ll @@ -77,11 +77,11 @@ define <8 x i32> @t1_vec_splat(<8 x i64> %x, <8 x i32> %nbits) { ret <8 x i32> %t5 } -define <8 x i32> @t2_vec_splat_undef(<8 x i64> %x, <8 x i32> %nbits) { -; CHECK-LABEL: @t2_vec_splat_undef( +define <8 x i32> @t2_vec_splat_poison(<8 x i64> %x, <8 x i32> %nbits) { +; CHECK-LABEL: @t2_vec_splat_poison( ; CHECK-NEXT: [[T0:%.*]] = zext <8 x i32> [[NBITS:%.*]] to <8 x i64> -; CHECK-NEXT: [[T1:%.*]] = lshr <8 x i64> , [[T0]] -; CHECK-NEXT: [[T2:%.*]] = add <8 x i32> [[NBITS]], +; CHECK-NEXT: [[T1:%.*]] = lshr <8 x i64> , [[T0]] +; CHECK-NEXT: [[T2:%.*]] = add <8 x i32> [[NBITS]], ; CHECK-NEXT: [[T3:%.*]] = and <8 x i64> [[T1]], [[X:%.*]] ; CHECK-NEXT: call void @use8xi64(<8 x i64> [[T0]]) ; CHECK-NEXT: call void @use8xi64(<8 x i64> [[T1]]) @@ -92,8 +92,8 @@ define <8 x i32> @t2_vec_splat_undef(<8 x i64> %x, <8 x i32> %nbits) { ; CHECK-NEXT: ret <8 x i32> [[T5]] ; %t0 = zext <8 x i32> %nbits to <8 x i64> - %t1 = lshr <8 x i64> , %t0 - %t2 = add <8 x i32> %nbits, + %t1 = lshr <8 x i64> , %t0 + %t2 = add <8 x i32> %nbits, %t3 = and <8 x i64> %t1, %x call void @use8xi64(<8 x i64> %t0) @@ -109,8 +109,8 @@ define <8 x i32> @t2_vec_splat_undef(<8 x i64> %x, <8 x i32> %nbits) { define <8 x i32> @t3_vec_nonsplat(<8 x i64> %x, <8 x i32> %nbits) { ; CHECK-LABEL: @t3_vec_nonsplat( ; CHECK-NEXT: [[T0:%.*]] = zext <8 x i32> [[NBITS:%.*]] to <8 x i64> -; CHECK-NEXT: [[T1:%.*]] = lshr <8 x i64> , [[T0]] -; CHECK-NEXT: [[T2:%.*]] = add <8 x i32> [[NBITS]], +; CHECK-NEXT: [[T1:%.*]] = lshr <8 x i64> , [[T0]] +; CHECK-NEXT: [[T2:%.*]] = add <8 x i32> [[NBITS]], ; CHECK-NEXT: [[T3:%.*]] = and <8 x i64> [[T1]], [[X:%.*]] ; CHECK-NEXT: call void @use8xi64(<8 x i64> [[T0]]) ; CHECK-NEXT: call void @use8xi64(<8 x i64> [[T1]]) @@ -121,8 +121,8 @@ define <8 x i32> @t3_vec_nonsplat(<8 x i64> %x, <8 x i32> %nbits) { ; CHECK-NEXT: ret <8 x i32> [[T5]] ; %t0 = zext <8 x i32> %nbits to <8 x i64> - %t1 = lshr <8 x i64> , %t0 - %t2 = add <8 x i32> %nbits, + %t1 = lshr <8 x i64> , %t0 + %t2 = add <8 x i32> %nbits, %t3 = and <8 x i64> %t1, %x call void @use8xi64(<8 x i64> %t0) diff --git a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-d.ll b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-d.ll index 1a977f67a6a5..b79ab7909752 100644 --- a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-d.ll +++ b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-d.ll @@ -85,12 +85,12 @@ define <8 x i32> @t1_vec_splat(<8 x i64> %x, <8 x i32> %nbits) { ret <8 x i32> %t6 } -define <8 x i32> @t2_vec_splat_undef(<8 x i64> %x, <8 x i32> %nbits) { -; CHECK-LABEL: @t2_vec_splat_undef( +define <8 x i32> @t2_vec_splat_poison(<8 x i64> %x, <8 x i32> %nbits) { +; CHECK-LABEL: @t2_vec_splat_poison( ; CHECK-NEXT: [[T0:%.*]] = zext <8 x i32> [[NBITS:%.*]] to <8 x i64> -; CHECK-NEXT: [[T1:%.*]] = shl <8 x i64> , [[T0]] +; CHECK-NEXT: [[T1:%.*]] = shl nsw <8 x i64> , [[T0]] ; CHECK-NEXT: [[T2:%.*]] = lshr <8 x i64> , [[T0]] -; CHECK-NEXT: [[T3:%.*]] = add <8 x i32> [[NBITS]], +; CHECK-NEXT: [[T3:%.*]] = add <8 x i32> [[NBITS]], ; CHECK-NEXT: [[T4:%.*]] = and <8 x i64> [[T2]], [[X:%.*]] ; CHECK-NEXT: call void @use8xi64(<8 x i64> [[T0]]) ; CHECK-NEXT: call void @use8xi64(<8 x i64> [[T1]]) @@ -102,9 +102,9 @@ define <8 x i32> @t2_vec_splat_undef(<8 x i64> %x, <8 x i32> %nbits) { ; CHECK-NEXT: ret <8 x i32> [[T6]] ; %t0 = zext <8 x i32> %nbits to <8 x i64> - %t1 = shl <8 x i64> , %t0 + %t1 = shl <8 x i64> , %t0 %t2 = lshr <8 x i64> %t1, %t0 - %t3 = add <8 x i32> %nbits, + %t3 = add <8 x i32> %nbits, %t4 = and <8 x i64> %t2, %x call void @use8xi64(<8 x i64> %t0) @@ -121,9 +121,9 @@ define <8 x i32> @t2_vec_splat_undef(<8 x i64> %x, <8 x i32> %nbits) { define <8 x i32> @t3_vec_nonsplat(<8 x i64> %x, <8 x i32> %nbits) { ; CHECK-LABEL: @t3_vec_nonsplat( ; CHECK-NEXT: [[T0:%.*]] = zext <8 x i32> [[NBITS:%.*]] to <8 x i64> -; CHECK-NEXT: [[T1:%.*]] = shl <8 x i64> , [[T0]] +; CHECK-NEXT: [[T1:%.*]] = shl nsw <8 x i64> , [[T0]] ; CHECK-NEXT: [[T2:%.*]] = lshr <8 x i64> , [[T0]] -; CHECK-NEXT: [[T3:%.*]] = add <8 x i32> [[NBITS]], +; CHECK-NEXT: [[T3:%.*]] = add <8 x i32> [[NBITS]], ; CHECK-NEXT: [[T4:%.*]] = and <8 x i64> [[T2]], [[X:%.*]] ; CHECK-NEXT: call void @use8xi64(<8 x i64> [[T0]]) ; CHECK-NEXT: call void @use8xi64(<8 x i64> [[T1]]) @@ -135,9 +135,9 @@ define <8 x i32> @t3_vec_nonsplat(<8 x i64> %x, <8 x i32> %nbits) { ; CHECK-NEXT: ret <8 x i32> [[T6]] ; %t0 = zext <8 x i32> %nbits to <8 x i64> - %t1 = shl <8 x i64> , %t0 + %t1 = shl <8 x i64> , %t0 %t2 = lshr <8 x i64> %t1, %t0 - %t3 = add <8 x i32> %nbits, + %t3 = add <8 x i32> %nbits, %t4 = and <8 x i64> %t2, %x call void @use8xi64(<8 x i64> %t0) diff --git a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-b.ll b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-b.ll index ddaef5f4b47c..4b955a894fcf 100644 --- a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-b.ll +++ b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-b.ll @@ -155,12 +155,12 @@ define <3 x i32> @t4_vec_nonsplat(<3 x i32> %x, <3 x i32> %nbits) { ret <3 x i32> %t5 } -define <3 x i32> @t5_vec_undef(<3 x i32> %x, <3 x i32> %nbits) { -; CHECK-LABEL: @t5_vec_undef( -; CHECK-NEXT: [[T1:%.*]] = shl <3 x i32> , [[NBITS:%.*]] -; CHECK-NEXT: [[T2:%.*]] = xor <3 x i32> [[T1]], +define <3 x i32> @t5_vec_poison(<3 x i32> %x, <3 x i32> %nbits) { +; CHECK-LABEL: @t5_vec_poison( +; CHECK-NEXT: [[T1:%.*]] = shl nsw <3 x i32> , [[NBITS:%.*]] +; CHECK-NEXT: [[T2:%.*]] = xor <3 x i32> [[T1]], ; CHECK-NEXT: [[T3:%.*]] = and <3 x i32> [[T2]], [[X:%.*]] -; CHECK-NEXT: [[T4:%.*]] = sub <3 x i32> , [[NBITS]] +; CHECK-NEXT: [[T4:%.*]] = sub <3 x i32> , [[NBITS]] ; CHECK-NEXT: call void @use3xi32(<3 x i32> [[NBITS]]) ; CHECK-NEXT: call void @use3xi32(<3 x i32> [[T1]]) ; CHECK-NEXT: call void @use3xi32(<3 x i32> [[T2]]) @@ -169,11 +169,11 @@ define <3 x i32> @t5_vec_undef(<3 x i32> %x, <3 x i32> %nbits) { ; CHECK-NEXT: [[T5:%.*]] = shl <3 x i32> [[X]], [[T4]] ; CHECK-NEXT: ret <3 x i32> [[T5]] ; - %t0 = add <3 x i32> %nbits, - %t1 = shl <3 x i32> , %t0 - %t2 = xor <3 x i32> %t1, + %t0 = add <3 x i32> %nbits, + %t1 = shl <3 x i32> , %t0 + %t2 = xor <3 x i32> %t1, %t3 = and <3 x i32> %t2, %x - %t4 = sub <3 x i32> , %nbits + %t4 = sub <3 x i32> , %nbits call void @use3xi32(<3 x i32> %t0) call void @use3xi32(<3 x i32> %t1) call void @use3xi32(<3 x i32> %t2) diff --git a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-c.ll b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-c.ll index c7747cfafcff..8428ef67d6b8 100644 --- a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-c.ll +++ b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-c.ll @@ -99,20 +99,20 @@ define <3 x i32> @t3_vec_nonsplat(<3 x i32> %x, <3 x i32> %nbits) { ret <3 x i32> %t3 } -define <3 x i32> @t4_vec_undef(<3 x i32> %x, <3 x i32> %nbits) { -; CHECK-LABEL: @t4_vec_undef( -; CHECK-NEXT: [[T0:%.*]] = lshr <3 x i32> , [[NBITS:%.*]] +define <3 x i32> @t4_vec_poison(<3 x i32> %x, <3 x i32> %nbits) { +; CHECK-LABEL: @t4_vec_poison( +; CHECK-NEXT: [[T0:%.*]] = lshr <3 x i32> , [[NBITS:%.*]] ; CHECK-NEXT: [[T1:%.*]] = and <3 x i32> [[T0]], [[X:%.*]] -; CHECK-NEXT: [[T2:%.*]] = add <3 x i32> [[NBITS]], +; CHECK-NEXT: [[T2:%.*]] = add <3 x i32> [[NBITS]], ; CHECK-NEXT: call void @use3xi32(<3 x i32> [[T0]]) ; CHECK-NEXT: call void @use3xi32(<3 x i32> [[T1]]) ; CHECK-NEXT: call void @use3xi32(<3 x i32> [[T2]]) ; CHECK-NEXT: [[T3:%.*]] = shl <3 x i32> [[X]], [[T2]] ; CHECK-NEXT: ret <3 x i32> [[T3]] ; - %t0 = lshr <3 x i32> , %nbits + %t0 = lshr <3 x i32> , %nbits %t1 = and <3 x i32> %t0, %x - %t2 = add <3 x i32> %nbits, + %t2 = add <3 x i32> %nbits, call void @use3xi32(<3 x i32> %t0) call void @use3xi32(<3 x i32> %t1) call void @use3xi32(<3 x i32> %t2) diff --git a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-d.ll b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-d.ll index 549729fe8b59..5d8ff9e9fb71 100644 --- a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-d.ll +++ b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-d.ll @@ -115,9 +115,9 @@ define <3 x i32> @t3_vec_nonsplat(<3 x i32> %x, <3 x i32> %nbits) { ret <3 x i32> %t4 } -define <3 x i32> @t4_vec_undef(<3 x i32> %x, <3 x i32> %nbits) { -; CHECK-LABEL: @t4_vec_undef( -; CHECK-NEXT: [[T0:%.*]] = shl <3 x i32> , [[NBITS:%.*]] +define <3 x i32> @t4_vec_poison(<3 x i32> %x, <3 x i32> %nbits) { +; CHECK-LABEL: @t4_vec_poison( +; CHECK-NEXT: [[T0:%.*]] = shl nsw <3 x i32> , [[NBITS:%.*]] ; CHECK-NEXT: [[T1:%.*]] = lshr <3 x i32> , [[NBITS]] ; CHECK-NEXT: [[T2:%.*]] = and <3 x i32> [[T1]], [[X:%.*]] ; CHECK-NEXT: call void @use3xi32(<3 x i32> [[T0]]) @@ -127,10 +127,10 @@ define <3 x i32> @t4_vec_undef(<3 x i32> %x, <3 x i32> %nbits) { ; CHECK-NEXT: [[T4:%.*]] = shl <3 x i32> [[X]], [[NBITS]] ; CHECK-NEXT: ret <3 x i32> [[T4]] ; - %t0 = shl <3 x i32> , %nbits + %t0 = shl <3 x i32> , %nbits %t1 = lshr <3 x i32> %t0, %nbits %t2 = and <3 x i32> %t1, %x - %t3 = add <3 x i32> %nbits, + %t3 = add <3 x i32> %nbits, call void @use3xi32(<3 x i32> %t0) call void @use3xi32(<3 x i32> %t1) call void @use3xi32(<3 x i32> %t2) diff --git a/llvm/test/Transforms/InstCombine/reuse-constant-from-select-in-icmp.ll b/llvm/test/Transforms/InstCombine/reuse-constant-from-select-in-icmp.ll index fd0d942ad840..301ead708a08 100644 --- a/llvm/test/Transforms/InstCombine/reuse-constant-from-select-in-icmp.ll +++ b/llvm/test/Transforms/InstCombine/reuse-constant-from-select-in-icmp.ll @@ -102,36 +102,36 @@ define <2 x i32> @p7_vec_splat_sgt(<2 x i32> %x, <2 x i32> %y) { ret <2 x i32> %r } -; Vectors with undef +; Vectors with poison -define <2 x i32> @p8_vec_nonsplat_undef0(<2 x i32> %x, <2 x i32> %y) { -; CHECK-LABEL: @p8_vec_nonsplat_undef0( +define <2 x i32> @p8_vec_nonsplat_poison0(<2 x i32> %x, <2 x i32> %y) { +; CHECK-LABEL: @p8_vec_nonsplat_poison0( ; CHECK-NEXT: [[T_INV:%.*]] = icmp ugt <2 x i32> [[X:%.*]], ; CHECK-NEXT: [[R:%.*]] = select <2 x i1> [[T_INV]], <2 x i32> , <2 x i32> [[Y:%.*]] ; CHECK-NEXT: ret <2 x i32> [[R]] ; - %t = icmp ult <2 x i32> %x, + %t = icmp ult <2 x i32> %x, %r = select <2 x i1> %t, <2 x i32> %y, <2 x i32> ret <2 x i32> %r } -define <2 x i32> @p9_vec_nonsplat_undef1(<2 x i32> %x, <2 x i32> %y) { -; CHECK-LABEL: @p9_vec_nonsplat_undef1( +define <2 x i32> @p9_vec_nonsplat_poison1(<2 x i32> %x, <2 x i32> %y) { +; CHECK-LABEL: @p9_vec_nonsplat_poison1( ; CHECK-NEXT: [[T_INV:%.*]] = icmp ugt <2 x i32> [[X:%.*]], -; CHECK-NEXT: [[R:%.*]] = select <2 x i1> [[T_INV]], <2 x i32> , <2 x i32> [[Y:%.*]] +; CHECK-NEXT: [[R:%.*]] = select <2 x i1> [[T_INV]], <2 x i32> , <2 x i32> [[Y:%.*]] ; CHECK-NEXT: ret <2 x i32> [[R]] ; %t = icmp ult <2 x i32> %x, - %r = select <2 x i1> %t, <2 x i32> %y, <2 x i32> + %r = select <2 x i1> %t, <2 x i32> %y, <2 x i32> ret <2 x i32> %r } -define <2 x i32> @p10_vec_nonsplat_undef2(<2 x i32> %x, <2 x i32> %y) { -; CHECK-LABEL: @p10_vec_nonsplat_undef2( +define <2 x i32> @p10_vec_nonsplat_poison2(<2 x i32> %x, <2 x i32> %y) { +; CHECK-LABEL: @p10_vec_nonsplat_poison2( ; CHECK-NEXT: [[T_INV:%.*]] = icmp ugt <2 x i32> [[X:%.*]], -; CHECK-NEXT: [[R:%.*]] = select <2 x i1> [[T_INV]], <2 x i32> , <2 x i32> [[Y:%.*]] +; CHECK-NEXT: [[R:%.*]] = select <2 x i1> [[T_INV]], <2 x i32> , <2 x i32> [[Y:%.*]] ; CHECK-NEXT: ret <2 x i32> [[R]] ; - %t = icmp ult <2 x i32> %x, - %r = select <2 x i1> %t, <2 x i32> %y, <2 x i32> + %t = icmp ult <2 x i32> %x, + %r = select <2 x i1> %t, <2 x i32> %y, <2 x i32> ret <2 x i32> %r } diff --git a/llvm/test/Transforms/InstCombine/rotate.ll b/llvm/test/Transforms/InstCombine/rotate.ll index 6c70c791fd88..eec623e2f193 100644 --- a/llvm/test/Transforms/InstCombine/rotate.ll +++ b/llvm/test/Transforms/InstCombine/rotate.ll @@ -65,24 +65,24 @@ define <2 x i16> @rotl_v2i16_constant_splat(<2 x i16> %x) { ret <2 x i16> %r } -define <2 x i16> @rotl_v2i16_constant_splat_undef0(<2 x i16> %x) { -; CHECK-LABEL: @rotl_v2i16_constant_splat_undef0( +define <2 x i16> @rotl_v2i16_constant_splat_poison0(<2 x i16> %x) { +; CHECK-LABEL: @rotl_v2i16_constant_splat_poison0( ; CHECK-NEXT: [[R:%.*]] = call <2 x i16> @llvm.fshl.v2i16(<2 x i16> [[X:%.*]], <2 x i16> [[X]], <2 x i16> ) ; CHECK-NEXT: ret <2 x i16> [[R]] ; - %shl = shl <2 x i16> %x, + %shl = shl <2 x i16> %x, %shr = lshr <2 x i16> %x, %r = or <2 x i16> %shl, %shr ret <2 x i16> %r } -define <2 x i16> @rotl_v2i16_constant_splat_undef1(<2 x i16> %x) { -; CHECK-LABEL: @rotl_v2i16_constant_splat_undef1( +define <2 x i16> @rotl_v2i16_constant_splat_poison1(<2 x i16> %x) { +; CHECK-LABEL: @rotl_v2i16_constant_splat_poison1( ; CHECK-NEXT: [[R:%.*]] = call <2 x i16> @llvm.fshl.v2i16(<2 x i16> [[X:%.*]], <2 x i16> [[X]], <2 x i16> ) ; CHECK-NEXT: ret <2 x i16> [[R]] ; %shl = shl <2 x i16> %x, - %shr = lshr <2 x i16> %x, + %shr = lshr <2 x i16> %x, %r = or <2 x i16> %shl, %shr ret <2 x i16> %r } @@ -100,30 +100,30 @@ define <2 x i17> @rotr_v2i17_constant_splat(<2 x i17> %x) { ret <2 x i17> %r } -define <2 x i17> @rotr_v2i17_constant_splat_undef0(<2 x i17> %x) { -; CHECK-LABEL: @rotr_v2i17_constant_splat_undef0( +define <2 x i17> @rotr_v2i17_constant_splat_poison0(<2 x i17> %x) { +; CHECK-LABEL: @rotr_v2i17_constant_splat_poison0( ; CHECK-NEXT: [[R:%.*]] = call <2 x i17> @llvm.fshl.v2i17(<2 x i17> [[X:%.*]], <2 x i17> [[X]], <2 x i17> ) ; CHECK-NEXT: ret <2 x i17> [[R]] ; - %shl = shl <2 x i17> %x, - %shr = lshr <2 x i17> %x, + %shl = shl <2 x i17> %x, + %shr = lshr <2 x i17> %x, %r = or <2 x i17> %shr, %shl ret <2 x i17> %r } -define <2 x i17> @rotr_v2i17_constant_splat_undef1(<2 x i17> %x) { -; CHECK-LABEL: @rotr_v2i17_constant_splat_undef1( +define <2 x i17> @rotr_v2i17_constant_splat_poison1(<2 x i17> %x) { +; CHECK-LABEL: @rotr_v2i17_constant_splat_poison1( ; CHECK-NEXT: [[R:%.*]] = call <2 x i17> @llvm.fshl.v2i17(<2 x i17> [[X:%.*]], <2 x i17> [[X]], <2 x i17> ) ; CHECK-NEXT: ret <2 x i17> [[R]] ; - %shl = shl <2 x i17> %x, - %shr = lshr <2 x i17> %x, + %shl = shl <2 x i17> %x, + %shr = lshr <2 x i17> %x, %r = or <2 x i17> %shr, %shl ret <2 x i17> %r } ; Allow arbitrary shift constants. -; Support undef elements. +; Support poison elements. define <2 x i32> @rotr_v2i32_constant_nonsplat(<2 x i32> %x) { ; CHECK-LABEL: @rotr_v2i32_constant_nonsplat( @@ -136,17 +136,6 @@ define <2 x i32> @rotr_v2i32_constant_nonsplat(<2 x i32> %x) { ret <2 x i32> %r } -define <2 x i32> @rotr_v2i32_constant_nonsplat_undef0(<2 x i32> %x) { -; CHECK-LABEL: @rotr_v2i32_constant_nonsplat_undef0( -; CHECK-NEXT: [[R:%.*]] = call <2 x i32> @llvm.fshl.v2i32(<2 x i32> [[X:%.*]], <2 x i32> [[X]], <2 x i32> ) -; CHECK-NEXT: ret <2 x i32> [[R]] -; - %shl = shl <2 x i32> %x, - %shr = lshr <2 x i32> %x, - %r = or <2 x i32> %shl, %shr - ret <2 x i32> %r -} - define <2 x i32> @rotr_v2i32_constant_nonsplat_poison0(<2 x i32> %x) { ; CHECK-LABEL: @rotr_v2i32_constant_nonsplat_poison0( ; CHECK-NEXT: [[R:%.*]] = call <2 x i32> @llvm.fshl.v2i32(<2 x i32> [[X:%.*]], <2 x i32> [[X]], <2 x i32> ) @@ -158,13 +147,13 @@ define <2 x i32> @rotr_v2i32_constant_nonsplat_poison0(<2 x i32> %x) { ret <2 x i32> %r } -define <2 x i32> @rotr_v2i32_constant_nonsplat_undef1(<2 x i32> %x) { -; CHECK-LABEL: @rotr_v2i32_constant_nonsplat_undef1( +define <2 x i32> @rotr_v2i32_constant_nonsplat_poison1(<2 x i32> %x) { +; CHECK-LABEL: @rotr_v2i32_constant_nonsplat_poison1( ; CHECK-NEXT: [[R:%.*]] = call <2 x i32> @llvm.fshl.v2i32(<2 x i32> [[X:%.*]], <2 x i32> [[X]], <2 x i32> ) ; CHECK-NEXT: ret <2 x i32> [[R]] ; %shl = shl <2 x i32> %x, - %shr = lshr <2 x i32> %x, + %shr = lshr <2 x i32> %x, %r = or <2 x i32> %shl, %shr ret <2 x i32> %r } @@ -180,13 +169,13 @@ define <2 x i36> @rotl_v2i36_constant_nonsplat(<2 x i36> %x) { ret <2 x i36> %r } -define <3 x i36> @rotl_v3i36_constant_nonsplat_undef0(<3 x i36> %x) { -; CHECK-LABEL: @rotl_v3i36_constant_nonsplat_undef0( -; CHECK-NEXT: [[R:%.*]] = call <3 x i36> @llvm.fshl.v3i36(<3 x i36> [[X:%.*]], <3 x i36> [[X]], <3 x i36> ) +define <3 x i36> @rotl_v3i36_constant_nonsplat_poison0(<3 x i36> %x) { +; CHECK-LABEL: @rotl_v3i36_constant_nonsplat_poison0( +; CHECK-NEXT: [[R:%.*]] = call <3 x i36> @llvm.fshl.v3i36(<3 x i36> [[X:%.*]], <3 x i36> [[X]], <3 x i36> ) ; CHECK-NEXT: ret <3 x i36> [[R]] ; - %shl = shl <3 x i36> %x, - %shr = lshr <3 x i36> %x, + %shl = shl <3 x i36> %x, + %shr = lshr <3 x i36> %x, %r = or <3 x i36> %shl, %shr ret <3 x i36> %r } diff --git a/llvm/test/Transforms/InstCombine/saturating-add-sub.ll b/llvm/test/Transforms/InstCombine/saturating-add-sub.ll index c1bb6941d456..57977a72cd08 100644 --- a/llvm/test/Transforms/InstCombine/saturating-add-sub.ll +++ b/llvm/test/Transforms/InstCombine/saturating-add-sub.ll @@ -559,14 +559,14 @@ define <2 x i8> @test_simplify_decrement_vec(<2 x i8> %a) { ret <2 x i8> %i2 } -define <2 x i8> @test_simplify_decrement_vec_undef(<2 x i8> %a) { -; CHECK-LABEL: @test_simplify_decrement_vec_undef( +define <2 x i8> @test_simplify_decrement_vec_poison(<2 x i8> %a) { +; CHECK-LABEL: @test_simplify_decrement_vec_poison( ; CHECK-NEXT: [[I2:%.*]] = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> [[A:%.*]], <2 x i8> ) ; CHECK-NEXT: ret <2 x i8> [[I2]] ; %i = icmp eq <2 x i8> %a, %i1 = sub <2 x i8> %a, - %i2 = select <2 x i1> %i, <2 x i8> , <2 x i8> %i1 + %i2 = select <2 x i1> %i, <2 x i8> , <2 x i8> %i1 ret <2 x i8> %i2 } @@ -1818,14 +1818,14 @@ define <4 x i32> @uadd_sat_constant_vec_commute(<4 x i32> %x) { define <4 x i32> @uadd_sat_constant_vec_commute_undefs(<4 x i32> %x) { ; CHECK-LABEL: @uadd_sat_constant_vec_commute_undefs( -; CHECK-NEXT: [[A:%.*]] = add <4 x i32> [[X:%.*]], -; CHECK-NEXT: [[C:%.*]] = icmp ult <4 x i32> [[X]], -; CHECK-NEXT: [[R:%.*]] = select <4 x i1> [[C]], <4 x i32> [[A]], <4 x i32> +; CHECK-NEXT: [[A:%.*]] = add <4 x i32> [[X:%.*]], +; CHECK-NEXT: [[C:%.*]] = icmp ult <4 x i32> [[X]], +; CHECK-NEXT: [[R:%.*]] = select <4 x i1> [[C]], <4 x i32> [[A]], <4 x i32> ; CHECK-NEXT: ret <4 x i32> [[R]] ; - %a = add <4 x i32> %x, - %c = icmp ult <4 x i32> %x, - %r = select <4 x i1> %c, <4 x i32> %a, <4 x i32> + %a = add <4 x i32> %x, + %c = icmp ult <4 x i32> %x, + %r = select <4 x i1> %c, <4 x i32> %a, <4 x i32> ret <4 x i32> %r } diff --git a/llvm/test/Transforms/InstCombine/select-of-bittest.ll b/llvm/test/Transforms/InstCombine/select-of-bittest.ll index a6f14cbfbfad..e3eb76de459e 100644 --- a/llvm/test/Transforms/InstCombine/select-of-bittest.ll +++ b/llvm/test/Transforms/InstCombine/select-of-bittest.ll @@ -80,19 +80,18 @@ define <2 x i32> @and_lshr_and_vec_v2(<2 x i32> %arg) { ret <2 x i32> %t4 } -define <3 x i32> @and_lshr_and_vec_undef(<3 x i32> %arg) { -; CHECK-LABEL: @and_lshr_and_vec_undef( +define <3 x i32> @and_lshr_and_vec_poison(<3 x i32> %arg) { +; CHECK-LABEL: @and_lshr_and_vec_poison( ; CHECK-NEXT: [[TMP1:%.*]] = and <3 x i32> [[ARG:%.*]], ; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <3 x i32> [[TMP1]], zeroinitializer ; CHECK-NEXT: [[T4:%.*]] = zext <3 x i1> [[TMP2]] to <3 x i32> ; CHECK-NEXT: ret <3 x i32> [[T4]] ; - %t = and <3 x i32> %arg, - %t1 = icmp eq <3 x i32> %t, - %t2 = lshr <3 x i32> %arg, - %t3 = and <3 x i32> %t2, - ; The second element of %t4 is poison because it is (undef ? poison : undef). - %t4 = select <3 x i1> %t1, <3 x i32> %t3, <3 x i32> + %t = and <3 x i32> %arg, + %t1 = icmp eq <3 x i32> %t, + %t2 = lshr <3 x i32> %arg, + %t3 = and <3 x i32> %t2, + %t4 = select <3 x i1> %t1, <3 x i32> %t3, <3 x i32> ret <3 x i32> %t4 } @@ -138,17 +137,17 @@ define <2 x i32> @and_and_vec(<2 x i32> %arg) { ret <2 x i32> %t3 } -define <3 x i32> @and_and_vec_undef(<3 x i32> %arg) { -; CHECK-LABEL: @and_and_vec_undef( -; CHECK-NEXT: [[TMP1:%.*]] = and <3 x i32> [[ARG:%.*]], +define <3 x i32> @and_and_vec_poison(<3 x i32> %arg) { +; CHECK-LABEL: @and_and_vec_poison( +; CHECK-NEXT: [[TMP1:%.*]] = and <3 x i32> [[ARG:%.*]], ; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <3 x i32> [[TMP1]], zeroinitializer ; CHECK-NEXT: [[T3:%.*]] = zext <3 x i1> [[TMP2]] to <3 x i32> ; CHECK-NEXT: ret <3 x i32> [[T3]] ; - %t = and <3 x i32> %arg, - %t1 = icmp eq <3 x i32> %t, - %t2 = and <3 x i32> %arg, - %t3 = select <3 x i1> %t1, <3 x i32> %t2, <3 x i32> + %t = and <3 x i32> %arg, + %t1 = icmp eq <3 x i32> %t, + %t2 = and <3 x i32> %arg, + %t3 = select <3 x i1> %t1, <3 x i32> %t2, <3 x i32> ret <3 x i32> %t3 } @@ -221,8 +220,8 @@ define <2 x i32> @f_var0_vec(<2 x i32> %arg, <2 x i32> %arg1) { ret <2 x i32> %t5 } -define <3 x i32> @f_var0_vec_undef(<3 x i32> %arg, <3 x i32> %arg1) { -; CHECK-LABEL: @f_var0_vec_undef( +define <3 x i32> @f_var0_vec_poison(<3 x i32> %arg, <3 x i32> %arg1) { +; CHECK-LABEL: @f_var0_vec_poison( ; CHECK-NEXT: [[TMP1:%.*]] = or <3 x i32> [[ARG1:%.*]], ; CHECK-NEXT: [[TMP2:%.*]] = and <3 x i32> [[TMP1]], [[ARG:%.*]] ; CHECK-NEXT: [[TMP3:%.*]] = icmp ne <3 x i32> [[TMP2]], zeroinitializer @@ -230,11 +229,11 @@ define <3 x i32> @f_var0_vec_undef(<3 x i32> %arg, <3 x i32> %arg1) { ; CHECK-NEXT: ret <3 x i32> [[T5]] ; %t = and <3 x i32> %arg, %arg1 - %t2 = icmp eq <3 x i32> %t, - %t3 = lshr <3 x i32> %arg, - %t4 = and <3 x i32> %t3, - ; The second element of %t5 is poison because it is (undef ? poison : undef). - %t5 = select <3 x i1> %t2, <3 x i32> %t4, <3 x i32> + %t2 = icmp eq <3 x i32> %t, + %t3 = lshr <3 x i32> %arg, + %t4 = and <3 x i32> %t3, + ; The second element of %t5 is poison because it is (poison ? poison : poison). + %t5 = select <3 x i1> %t2, <3 x i32> %t4, <3 x i32> ret <3 x i32> %t5 } @@ -284,8 +283,8 @@ define <2 x i32> @f_var1_vec(<2 x i32> %arg, <2 x i32> %arg1) { ret <2 x i32> %t4 } -define <3 x i32> @f_var1_vec_undef(<3 x i32> %arg, <3 x i32> %arg1) { -; CHECK-LABEL: @f_var1_vec_undef( +define <3 x i32> @f_var1_vec_poison(<3 x i32> %arg, <3 x i32> %arg1) { +; CHECK-LABEL: @f_var1_vec_poison( ; CHECK-NEXT: [[TMP1:%.*]] = or <3 x i32> [[ARG1:%.*]], ; CHECK-NEXT: [[TMP2:%.*]] = and <3 x i32> [[TMP1]], [[ARG:%.*]] ; CHECK-NEXT: [[TMP3:%.*]] = icmp ne <3 x i32> [[TMP2]], zeroinitializer @@ -293,9 +292,9 @@ define <3 x i32> @f_var1_vec_undef(<3 x i32> %arg, <3 x i32> %arg1) { ; CHECK-NEXT: ret <3 x i32> [[T4]] ; %t = and <3 x i32> %arg, %arg1 - %t2 = icmp eq <3 x i32> %t, - %t3 = and <3 x i32> %arg, - %t4 = select <3 x i1> %t2, <3 x i32> %t3, <3 x i32> + %t2 = icmp eq <3 x i32> %t, + %t3 = and <3 x i32> %arg, + %t4 = select <3 x i1> %t2, <3 x i32> %t3, <3 x i32> ret <3 x i32> %t4 } @@ -354,20 +353,20 @@ define <2 x i32> @f_var2_vec(<2 x i32> %arg, <2 x i32> %arg1) { ret <2 x i32> %t5 } -define <3 x i32> @f_var2_vec_undef(<3 x i32> %arg, <3 x i32> %arg1) { -; CHECK-LABEL: @f_var2_vec_undef( -; CHECK-NEXT: [[T:%.*]] = and <3 x i32> [[ARG:%.*]], -; CHECK-NEXT: [[T2:%.*]] = icmp eq <3 x i32> [[T]], +define <3 x i32> @f_var2_vec_poison(<3 x i32> %arg, <3 x i32> %arg1) { +; CHECK-LABEL: @f_var2_vec_poison( +; CHECK-NEXT: [[T:%.*]] = and <3 x i32> [[ARG:%.*]], +; CHECK-NEXT: [[T2:%.*]] = icmp eq <3 x i32> [[T]], ; CHECK-NEXT: [[T3:%.*]] = lshr <3 x i32> [[ARG]], [[ARG1:%.*]] -; CHECK-NEXT: [[T4:%.*]] = and <3 x i32> [[T3]], -; CHECK-NEXT: [[T5:%.*]] = select <3 x i1> [[T2]], <3 x i32> [[T4]], <3 x i32> +; CHECK-NEXT: [[T4:%.*]] = and <3 x i32> [[T3]], +; CHECK-NEXT: [[T5:%.*]] = select <3 x i1> [[T2]], <3 x i32> [[T4]], <3 x i32> ; CHECK-NEXT: ret <3 x i32> [[T5]] ; - %t = and <3 x i32> %arg, - %t2 = icmp eq <3 x i32> %t, + %t = and <3 x i32> %arg, + %t2 = icmp eq <3 x i32> %t, %t3 = lshr <3 x i32> %arg, %arg1 - %t4 = and <3 x i32> %t3, - %t5 = select <3 x i1> %t2, <3 x i32> %t4, <3 x i32> + %t4 = and <3 x i32> %t3, + %t5 = select <3 x i1> %t2, <3 x i32> %t4, <3 x i32> ret <3 x i32> %t5 } @@ -427,20 +426,20 @@ define <2 x i32> @f_var3_splatvec(<2 x i32> %arg, <2 x i32> %arg1, <2 x i32> %ar ret <2 x i32> %t6 } -define <3 x i32> @f_var3_vec_undef(<3 x i32> %arg, <3 x i32> %arg1, <3 x i32> %arg2) { -; CHECK-LABEL: @f_var3_vec_undef( +define <3 x i32> @f_var3_vec_poison(<3 x i32> %arg, <3 x i32> %arg1, <3 x i32> %arg2) { +; CHECK-LABEL: @f_var3_vec_poison( ; CHECK-NEXT: [[T:%.*]] = and <3 x i32> [[ARG:%.*]], [[ARG1:%.*]] -; CHECK-NEXT: [[T3:%.*]] = icmp eq <3 x i32> [[T]], +; CHECK-NEXT: [[T3:%.*]] = icmp eq <3 x i32> [[T]], ; CHECK-NEXT: [[T4:%.*]] = lshr <3 x i32> [[ARG]], [[ARG2:%.*]] -; CHECK-NEXT: [[T5:%.*]] = and <3 x i32> [[T4]], -; CHECK-NEXT: [[T6:%.*]] = select <3 x i1> [[T3]], <3 x i32> [[T5]], <3 x i32> +; CHECK-NEXT: [[T5:%.*]] = and <3 x i32> [[T4]], +; CHECK-NEXT: [[T6:%.*]] = select <3 x i1> [[T3]], <3 x i32> [[T5]], <3 x i32> ; CHECK-NEXT: ret <3 x i32> [[T6]] ; %t = and <3 x i32> %arg, %arg1 - %t3 = icmp eq <3 x i32> %t, + %t3 = icmp eq <3 x i32> %t, %t4 = lshr <3 x i32> %arg, %arg2 - %t5 = and <3 x i32> %t4, - %t6 = select <3 x i1> %t3, <3 x i32> %t5, <3 x i32> + %t5 = and <3 x i32> %t4, + %t6 = select <3 x i1> %t3, <3 x i32> %t5, <3 x i32> ret <3 x i32> %t6 } diff --git a/llvm/test/Transforms/InstCombine/select.ll b/llvm/test/Transforms/InstCombine/select.ll index bd8145ab2a35..8654691c6f87 100644 --- a/llvm/test/Transforms/InstCombine/select.ll +++ b/llvm/test/Transforms/InstCombine/select.ll @@ -3109,45 +3109,46 @@ define <4 x i32> @mul_select_eq_zero_vector(<4 x i32> %x, <4 x i32> %y) { } ; Check that a select is folded into multiplication if condition's operand -; is a vector consisting of zeros and undefs. -; select ( x == {0, undef, ...}), 0, x * y --> freeze(y) * x -define <2 x i32> @mul_select_eq_undef_vector(<2 x i32> %x, <2 x i32> %y) { -; CHECK-LABEL: @mul_select_eq_undef_vector( -; CHECK-NEXT: [[Y_FR:%.*]] = freeze <2 x i32> [[Y:%.*]] +; is a vector consisting of zeros and poisons. +; select ( x == {0, poison, ...}), 0, x * y --> freeze(y) * x +define <2 x i32> @mul_select_eq_poison_vector(<2 x i32> %x, <2 x i32> %y) { +; CHECK-LABEL: @mul_select_eq_poison_vector( +; CHECK-NEXT: [[C:%.*]] = icmp eq <2 x i32> [[Y_FR:%.*]], ; CHECK-NEXT: [[M:%.*]] = mul <2 x i32> [[Y_FR]], [[X:%.*]] -; CHECK-NEXT: ret <2 x i32> [[M]] +; CHECK-NEXT: [[R:%.*]] = select <2 x i1> [[C]], <2 x i32> , <2 x i32> [[M]] +; CHECK-NEXT: ret <2 x i32> [[R]] ; - %c = icmp eq <2 x i32> %x, + %c = icmp eq <2 x i32> %x, %m = mul <2 x i32> %x, %y %r = select <2 x i1> %c, <2 x i32> , <2 x i32> %m ret <2 x i32> %r } ; Check that a select is folded into multiplication if other select's operand -; is a vector consisting of zeros and undefs. -; select ( x == 0), {0, undef, ...}, x * y --> freeze(y) * x -define <2 x i32> @mul_select_eq_zero_sel_undef_vector(<2 x i32> %x, <2 x i32> %y) { -; CHECK-LABEL: @mul_select_eq_zero_sel_undef_vector( +; is a vector consisting of zeros and poisons. +; select ( x == 0), {0, poison, ...}, x * y --> freeze(y) * x +define <2 x i32> @mul_select_eq_zero_sel_poison_vector(<2 x i32> %x, <2 x i32> %y) { +; CHECK-LABEL: @mul_select_eq_zero_sel_poison_vector( ; CHECK-NEXT: [[Y_FR:%.*]] = freeze <2 x i32> [[Y:%.*]] ; CHECK-NEXT: [[M:%.*]] = mul <2 x i32> [[Y_FR]], [[X:%.*]] ; CHECK-NEXT: ret <2 x i32> [[M]] ; %c = icmp eq <2 x i32> %x, zeroinitializer %m = mul <2 x i32> %x, %y - %r = select <2 x i1> %c, <2 x i32> , <2 x i32> %m + %r = select <2 x i1> %c, <2 x i32> , <2 x i32> %m ret <2 x i32> %r } ; Negative test: select should not be folded into mul because ; condition's operand and select's operand do not merge into zero vector. -define <2 x i32> @mul_select_eq_undef_vector_not_merging_to_zero(<2 x i32> %x, <2 x i32> %y) { -; CHECK-LABEL: @mul_select_eq_undef_vector_not_merging_to_zero( -; CHECK-NEXT: [[C:%.*]] = icmp eq <2 x i32> [[X:%.*]], +define <2 x i32> @mul_select_eq_poison_vector_not_merging_to_zero(<2 x i32> %x, <2 x i32> %y) { +; CHECK-LABEL: @mul_select_eq_poison_vector_not_merging_to_zero( +; CHECK-NEXT: [[C:%.*]] = icmp eq <2 x i32> [[X:%.*]], ; CHECK-NEXT: [[M:%.*]] = mul <2 x i32> [[X]], [[Y:%.*]] ; CHECK-NEXT: [[R:%.*]] = select <2 x i1> [[C]], <2 x i32> , <2 x i32> [[M]] ; CHECK-NEXT: ret <2 x i32> [[R]] ; - %c = icmp eq <2 x i32> %x, + %c = icmp eq <2 x i32> %x, %m = mul <2 x i32> %x, %y %r = select <2 x i1> %c, <2 x i32> , <2 x i32> %m ret <2 x i32> %r diff --git a/llvm/test/Transforms/InstCombine/select_meta.ll b/llvm/test/Transforms/InstCombine/select_meta.ll index aa794e82e0fd..3898fd9fa1f5 100644 --- a/llvm/test/Transforms/InstCombine/select_meta.ll +++ b/llvm/test/Transforms/InstCombine/select_meta.ll @@ -301,15 +301,15 @@ define <2 x i32> @not_cond_vec(<2 x i1> %c, <2 x i32> %tv, <2 x i32> %fv) { ret <2 x i32> %r } -; Should match vector 'not' with undef element. +; Should match vector 'not' with poison element. ; The condition is inverted, and the select ops are swapped. The metadata should be swapped. -define <2 x i32> @not_cond_vec_undef(<2 x i1> %c, <2 x i32> %tv, <2 x i32> %fv) { -; CHECK-LABEL: @not_cond_vec_undef( +define <2 x i32> @not_cond_vec_poison(<2 x i1> %c, <2 x i32> %tv, <2 x i32> %fv) { +; CHECK-LABEL: @not_cond_vec_poison( ; CHECK-NEXT: [[R:%.*]] = select <2 x i1> [[C:%.*]], <2 x i32> [[FV:%.*]], <2 x i32> [[TV:%.*]], !prof [[PROF1]] ; CHECK-NEXT: ret <2 x i32> [[R]] ; - %notc = xor <2 x i1> %c, + %notc = xor <2 x i1> %c, %r = select <2 x i1> %notc, <2 x i32> %tv, <2 x i32> %fv, !prof !1 ret <2 x i32> %r } diff --git a/llvm/test/Transforms/InstCombine/set-lowbits-mask-canonicalize.ll b/llvm/test/Transforms/InstCombine/set-lowbits-mask-canonicalize.ll index 3ee0224eb1d0..a3c8d3393d04 100644 --- a/llvm/test/Transforms/InstCombine/set-lowbits-mask-canonicalize.ll +++ b/llvm/test/Transforms/InstCombine/set-lowbits-mask-canonicalize.ll @@ -196,36 +196,36 @@ define <2 x i32> @shl_add_vec(<2 x i32> %NBits) { ret <2 x i32> %ret } -define <3 x i32> @shl_add_vec_undef0(<3 x i32> %NBits) { -; CHECK-LABEL: @shl_add_vec_undef0( +define <3 x i32> @shl_add_vec_poison0(<3 x i32> %NBits) { +; CHECK-LABEL: @shl_add_vec_poison0( ; CHECK-NEXT: [[NOTMASK:%.*]] = shl nsw <3 x i32> , [[NBITS:%.*]] ; CHECK-NEXT: [[RET:%.*]] = xor <3 x i32> [[NOTMASK]], ; CHECK-NEXT: ret <3 x i32> [[RET]] ; - %setbit = shl <3 x i32> , %NBits + %setbit = shl <3 x i32> , %NBits %ret = add <3 x i32> %setbit, ret <3 x i32> %ret } -define <3 x i32> @shl_add_vec_undef1(<3 x i32> %NBits) { -; CHECK-LABEL: @shl_add_vec_undef1( +define <3 x i32> @shl_add_vec_poison1(<3 x i32> %NBits) { +; CHECK-LABEL: @shl_add_vec_poison1( ; CHECK-NEXT: [[NOTMASK:%.*]] = shl nsw <3 x i32> , [[NBITS:%.*]] ; CHECK-NEXT: [[RET:%.*]] = xor <3 x i32> [[NOTMASK]], ; CHECK-NEXT: ret <3 x i32> [[RET]] ; %setbit = shl <3 x i32> , %NBits - %ret = add <3 x i32> %setbit, + %ret = add <3 x i32> %setbit, ret <3 x i32> %ret } -define <3 x i32> @shl_add_vec_undef2(<3 x i32> %NBits) { -; CHECK-LABEL: @shl_add_vec_undef2( +define <3 x i32> @shl_add_vec_poison2(<3 x i32> %NBits) { +; CHECK-LABEL: @shl_add_vec_poison2( ; CHECK-NEXT: [[NOTMASK:%.*]] = shl nsw <3 x i32> , [[NBITS:%.*]] ; CHECK-NEXT: [[RET:%.*]] = xor <3 x i32> [[NOTMASK]], ; CHECK-NEXT: ret <3 x i32> [[RET]] ; - %setbit = shl <3 x i32> , %NBits - %ret = add <3 x i32> %setbit, + %setbit = shl <3 x i32> , %NBits + %ret = add <3 x i32> %setbit, ret <3 x i32> %ret } diff --git a/llvm/test/Transforms/InstCombine/sext.ll b/llvm/test/Transforms/InstCombine/sext.ll index e3b6058ce7f8..6d263cfcda05 100644 --- a/llvm/test/Transforms/InstCombine/sext.ll +++ b/llvm/test/Transforms/InstCombine/sext.ll @@ -167,39 +167,39 @@ define <2 x i32> @test10_vec_nonuniform(<2 x i32> %i) { ret <2 x i32> %D } -define <2 x i32> @test10_vec_undef0(<2 x i32> %i) { -; CHECK-LABEL: @test10_vec_undef0( -; CHECK-NEXT: [[D1:%.*]] = shl <2 x i32> [[I:%.*]], -; CHECK-NEXT: [[D:%.*]] = ashr exact <2 x i32> [[D1]], +define <2 x i32> @test10_vec_poison0(<2 x i32> %i) { +; CHECK-LABEL: @test10_vec_poison0( +; CHECK-NEXT: [[D1:%.*]] = shl <2 x i32> [[I:%.*]], +; CHECK-NEXT: [[D:%.*]] = ashr exact <2 x i32> [[D1]], ; CHECK-NEXT: ret <2 x i32> [[D]] ; %A = trunc <2 x i32> %i to <2 x i8> %B = shl <2 x i8> %A, - %C = ashr <2 x i8> %B, + %C = ashr <2 x i8> %B, %D = sext <2 x i8> %C to <2 x i32> ret <2 x i32> %D } -define <2 x i32> @test10_vec_undef1(<2 x i32> %i) { -; CHECK-LABEL: @test10_vec_undef1( +define <2 x i32> @test10_vec_poison1(<2 x i32> %i) { +; CHECK-LABEL: @test10_vec_poison1( ; CHECK-NEXT: [[D1:%.*]] = shl <2 x i32> [[I:%.*]], ; CHECK-NEXT: [[D:%.*]] = ashr exact <2 x i32> [[D1]], ; CHECK-NEXT: ret <2 x i32> [[D]] ; %A = trunc <2 x i32> %i to <2 x i8> - %B = shl <2 x i8> %A, + %B = shl <2 x i8> %A, %C = ashr <2 x i8> %B, %D = sext <2 x i8> %C to <2 x i32> ret <2 x i32> %D } -define <2 x i32> @test10_vec_undef2(<2 x i32> %i) { -; CHECK-LABEL: @test10_vec_undef2( -; CHECK-NEXT: [[D1:%.*]] = shl <2 x i32> [[I:%.*]], -; CHECK-NEXT: [[D:%.*]] = ashr exact <2 x i32> [[D1]], +define <2 x i32> @test10_vec_poison2(<2 x i32> %i) { +; CHECK-LABEL: @test10_vec_poison2( +; CHECK-NEXT: [[D1:%.*]] = shl <2 x i32> [[I:%.*]], +; CHECK-NEXT: [[D:%.*]] = ashr exact <2 x i32> [[D1]], ; CHECK-NEXT: ret <2 x i32> [[D]] ; %A = trunc <2 x i32> %i to <2 x i8> - %B = shl <2 x i8> %A, - %C = ashr <2 x i8> %B, + %B = shl <2 x i8> %A, + %C = ashr <2 x i8> %B, %D = sext <2 x i8> %C to <2 x i32> ret <2 x i32> %D } diff --git a/llvm/test/Transforms/InstCombine/shift-amount-reassociation-in-bittest.ll b/llvm/test/Transforms/InstCombine/shift-amount-reassociation-in-bittest.ll index 0262db1a01e5..96d429c62a88 100644 --- a/llvm/test/Transforms/InstCombine/shift-amount-reassociation-in-bittest.ll +++ b/llvm/test/Transforms/InstCombine/shift-amount-reassociation-in-bittest.ll @@ -143,34 +143,34 @@ define <2 x i1> @t8_const_lshr_shl_ne_vec_nonsplat(<2 x i32> %x, <2 x i32> %y) { %t3 = icmp ne <2 x i32> %t2, ret <2 x i1> %t3 } -define <3 x i1> @t9_const_lshr_shl_ne_vec_undef0(<3 x i32> %x, <3 x i32> %y) { -; CHECK-LABEL: @t9_const_lshr_shl_ne_vec_undef0( -; CHECK-NEXT: [[TMP1:%.*]] = lshr <3 x i32> [[X:%.*]], +define <3 x i1> @t9_const_lshr_shl_ne_vec_poison0(<3 x i32> %x, <3 x i32> %y) { +; CHECK-LABEL: @t9_const_lshr_shl_ne_vec_poison0( +; CHECK-NEXT: [[TMP1:%.*]] = lshr <3 x i32> [[X:%.*]], ; CHECK-NEXT: [[TMP2:%.*]] = and <3 x i32> [[TMP1]], [[Y:%.*]] ; CHECK-NEXT: [[T3:%.*]] = icmp ne <3 x i32> [[TMP2]], zeroinitializer ; CHECK-NEXT: ret <3 x i1> [[T3]] ; - %t0 = lshr <3 x i32> %x, + %t0 = lshr <3 x i32> %x, %t1 = shl <3 x i32> %y, %t2 = and <3 x i32> %t1, %t0 %t3 = icmp ne <3 x i32> %t2, ret <3 x i1> %t3 } -define <3 x i1> @t10_const_lshr_shl_ne_vec_undef1(<3 x i32> %x, <3 x i32> %y) { -; CHECK-LABEL: @t10_const_lshr_shl_ne_vec_undef1( -; CHECK-NEXT: [[TMP1:%.*]] = lshr <3 x i32> [[X:%.*]], +define <3 x i1> @t10_const_lshr_shl_ne_vec_poison1(<3 x i32> %x, <3 x i32> %y) { +; CHECK-LABEL: @t10_const_lshr_shl_ne_vec_poison1( +; CHECK-NEXT: [[TMP1:%.*]] = lshr <3 x i32> [[X:%.*]], ; CHECK-NEXT: [[TMP2:%.*]] = and <3 x i32> [[TMP1]], [[Y:%.*]] ; CHECK-NEXT: [[T3:%.*]] = icmp ne <3 x i32> [[TMP2]], zeroinitializer ; CHECK-NEXT: ret <3 x i1> [[T3]] ; %t0 = lshr <3 x i32> %x, - %t1 = shl <3 x i32> %y, + %t1 = shl <3 x i32> %y, %t2 = and <3 x i32> %t1, %t0 %t3 = icmp ne <3 x i32> %t2, ret <3 x i1> %t3 } -define <3 x i1> @t11_const_lshr_shl_ne_vec_undef2(<3 x i32> %x, <3 x i32> %y) { -; CHECK-LABEL: @t11_const_lshr_shl_ne_vec_undef2( +define <3 x i1> @t11_const_lshr_shl_ne_vec_poison2(<3 x i32> %x, <3 x i32> %y) { +; CHECK-LABEL: @t11_const_lshr_shl_ne_vec_poison2( ; CHECK-NEXT: [[TMP1:%.*]] = lshr <3 x i32> [[X:%.*]], ; CHECK-NEXT: [[TMP2:%.*]] = and <3 x i32> [[TMP1]], [[Y:%.*]] ; CHECK-NEXT: [[T3:%.*]] = icmp ne <3 x i32> [[TMP2]], zeroinitializer @@ -179,59 +179,59 @@ define <3 x i1> @t11_const_lshr_shl_ne_vec_undef2(<3 x i32> %x, <3 x i32> %y) { %t0 = lshr <3 x i32> %x, %t1 = shl <3 x i32> %y, %t2 = and <3 x i32> %t1, %t0 - %t3 = icmp ne <3 x i32> %t2, + %t3 = icmp ne <3 x i32> %t2, ret <3 x i1> %t3 } -define <3 x i1> @t12_const_lshr_shl_ne_vec_undef3(<3 x i32> %x, <3 x i32> %y) { -; CHECK-LABEL: @t12_const_lshr_shl_ne_vec_undef3( -; CHECK-NEXT: [[TMP1:%.*]] = lshr <3 x i32> [[X:%.*]], +define <3 x i1> @t12_const_lshr_shl_ne_vec_poison3(<3 x i32> %x, <3 x i32> %y) { +; CHECK-LABEL: @t12_const_lshr_shl_ne_vec_poison3( +; CHECK-NEXT: [[TMP1:%.*]] = lshr <3 x i32> [[X:%.*]], ; CHECK-NEXT: [[TMP2:%.*]] = and <3 x i32> [[TMP1]], [[Y:%.*]] ; CHECK-NEXT: [[T3:%.*]] = icmp ne <3 x i32> [[TMP2]], zeroinitializer ; CHECK-NEXT: ret <3 x i1> [[T3]] ; - %t0 = lshr <3 x i32> %x, - %t1 = shl <3 x i32> %y, + %t0 = lshr <3 x i32> %x, + %t1 = shl <3 x i32> %y, %t2 = and <3 x i32> %t1, %t0 %t3 = icmp ne <3 x i32> %t2, ret <3 x i1> %t3 } -define <3 x i1> @t13_const_lshr_shl_ne_vec_undef4(<3 x i32> %x, <3 x i32> %y) { -; CHECK-LABEL: @t13_const_lshr_shl_ne_vec_undef4( -; CHECK-NEXT: [[TMP1:%.*]] = lshr <3 x i32> [[X:%.*]], +define <3 x i1> @t13_const_lshr_shl_ne_vec_poison4(<3 x i32> %x, <3 x i32> %y) { +; CHECK-LABEL: @t13_const_lshr_shl_ne_vec_poison4( +; CHECK-NEXT: [[TMP1:%.*]] = lshr <3 x i32> [[X:%.*]], ; CHECK-NEXT: [[TMP2:%.*]] = and <3 x i32> [[TMP1]], [[Y:%.*]] ; CHECK-NEXT: [[T3:%.*]] = icmp ne <3 x i32> [[TMP2]], zeroinitializer ; CHECK-NEXT: ret <3 x i1> [[T3]] ; %t0 = lshr <3 x i32> %x, - %t1 = shl <3 x i32> %y, + %t1 = shl <3 x i32> %y, %t2 = and <3 x i32> %t1, %t0 - %t3 = icmp ne <3 x i32> %t2, + %t3 = icmp ne <3 x i32> %t2, ret <3 x i1> %t3 } -define <3 x i1> @t14_const_lshr_shl_ne_vec_undef5(<3 x i32> %x, <3 x i32> %y) { -; CHECK-LABEL: @t14_const_lshr_shl_ne_vec_undef5( -; CHECK-NEXT: [[TMP1:%.*]] = lshr <3 x i32> [[X:%.*]], +define <3 x i1> @t14_const_lshr_shl_ne_vec_poison5(<3 x i32> %x, <3 x i32> %y) { +; CHECK-LABEL: @t14_const_lshr_shl_ne_vec_poison5( +; CHECK-NEXT: [[TMP1:%.*]] = lshr <3 x i32> [[X:%.*]], ; CHECK-NEXT: [[TMP2:%.*]] = and <3 x i32> [[TMP1]], [[Y:%.*]] ; CHECK-NEXT: [[T3:%.*]] = icmp ne <3 x i32> [[TMP2]], zeroinitializer ; CHECK-NEXT: ret <3 x i1> [[T3]] ; - %t0 = lshr <3 x i32> %x, + %t0 = lshr <3 x i32> %x, %t1 = shl <3 x i32> %y, %t2 = and <3 x i32> %t1, %t0 - %t3 = icmp ne <3 x i32> %t2, + %t3 = icmp ne <3 x i32> %t2, ret <3 x i1> %t3 } -define <3 x i1> @t15_const_lshr_shl_ne_vec_undef6(<3 x i32> %x, <3 x i32> %y) { -; CHECK-LABEL: @t15_const_lshr_shl_ne_vec_undef6( -; CHECK-NEXT: [[TMP1:%.*]] = lshr <3 x i32> [[X:%.*]], +define <3 x i1> @t15_const_lshr_shl_ne_vec_poison6(<3 x i32> %x, <3 x i32> %y) { +; CHECK-LABEL: @t15_const_lshr_shl_ne_vec_poison6( +; CHECK-NEXT: [[TMP1:%.*]] = lshr <3 x i32> [[X:%.*]], ; CHECK-NEXT: [[TMP2:%.*]] = and <3 x i32> [[TMP1]], [[Y:%.*]] ; CHECK-NEXT: [[T3:%.*]] = icmp ne <3 x i32> [[TMP2]], zeroinitializer ; CHECK-NEXT: ret <3 x i1> [[T3]] ; - %t0 = lshr <3 x i32> %x, - %t1 = shl <3 x i32> %y, + %t0 = lshr <3 x i32> %x, + %t1 = shl <3 x i32> %y, %t2 = and <3 x i32> %t1, %t0 - %t3 = icmp ne <3 x i32> %t2, + %t3 = icmp ne <3 x i32> %t2, ret <3 x i1> %t3 } diff --git a/llvm/test/Transforms/InstCombine/shift-amount-reassociation-with-truncation-ashr.ll b/llvm/test/Transforms/InstCombine/shift-amount-reassociation-with-truncation-ashr.ll index 84dd4c57ebc6..9efc30cc9d91 100644 --- a/llvm/test/Transforms/InstCombine/shift-amount-reassociation-with-truncation-ashr.ll +++ b/llvm/test/Transforms/InstCombine/shift-amount-reassociation-with-truncation-ashr.ll @@ -42,13 +42,13 @@ define <2 x i16> @t1_vec_splat(<2 x i32> %x, <2 x i16> %y) { ret <2 x i16> %t5 } -define <3 x i16> @t3_vec_nonsplat_undef0(<3 x i32> %x, <3 x i16> %y) { -; CHECK-LABEL: @t3_vec_nonsplat_undef0( -; CHECK-NEXT: [[TMP1:%.*]] = ashr <3 x i32> [[X:%.*]], +define <3 x i16> @t3_vec_nonsplat_poison0(<3 x i32> %x, <3 x i16> %y) { +; CHECK-LABEL: @t3_vec_nonsplat_poison0( +; CHECK-NEXT: [[TMP1:%.*]] = ashr <3 x i32> [[X:%.*]], ; CHECK-NEXT: [[T5:%.*]] = trunc <3 x i32> [[TMP1]] to <3 x i16> ; CHECK-NEXT: ret <3 x i16> [[T5]] ; - %t0 = sub <3 x i16> , %y + %t0 = sub <3 x i16> , %y %t1 = zext <3 x i16> %t0 to <3 x i32> %t2 = ashr <3 x i32> %x, %t1 %t3 = trunc <3 x i32> %t2 to <3 x i16> @@ -57,9 +57,9 @@ define <3 x i16> @t3_vec_nonsplat_undef0(<3 x i32> %x, <3 x i16> %y) { ret <3 x i16> %t5 } -define <3 x i16> @t4_vec_nonsplat_undef1(<3 x i32> %x, <3 x i16> %y) { -; CHECK-LABEL: @t4_vec_nonsplat_undef1( -; CHECK-NEXT: [[TMP1:%.*]] = ashr <3 x i32> [[X:%.*]], +define <3 x i16> @t4_vec_nonsplat_poison1(<3 x i32> %x, <3 x i16> %y) { +; CHECK-LABEL: @t4_vec_nonsplat_poison1( +; CHECK-NEXT: [[TMP1:%.*]] = ashr <3 x i32> [[X:%.*]], ; CHECK-NEXT: [[T5:%.*]] = trunc <3 x i32> [[TMP1]] to <3 x i16> ; CHECK-NEXT: ret <3 x i16> [[T5]] ; @@ -67,22 +67,22 @@ define <3 x i16> @t4_vec_nonsplat_undef1(<3 x i32> %x, <3 x i16> %y) { %t1 = zext <3 x i16> %t0 to <3 x i32> %t2 = ashr <3 x i32> %x, %t1 %t3 = trunc <3 x i32> %t2 to <3 x i16> - %t4 = add <3 x i16> %y, + %t4 = add <3 x i16> %y, %t5 = ashr <3 x i16> %t3, %t4 ret <3 x i16> %t5 } -define <3 x i16> @t5_vec_nonsplat_undef1(<3 x i32> %x, <3 x i16> %y) { -; CHECK-LABEL: @t5_vec_nonsplat_undef1( -; CHECK-NEXT: [[TMP1:%.*]] = ashr <3 x i32> [[X:%.*]], +define <3 x i16> @t5_vec_nonsplat_poison1(<3 x i32> %x, <3 x i16> %y) { +; CHECK-LABEL: @t5_vec_nonsplat_poison1( +; CHECK-NEXT: [[TMP1:%.*]] = ashr <3 x i32> [[X:%.*]], ; CHECK-NEXT: [[T5:%.*]] = trunc <3 x i32> [[TMP1]] to <3 x i16> ; CHECK-NEXT: ret <3 x i16> [[T5]] ; - %t0 = sub <3 x i16> , %y + %t0 = sub <3 x i16> , %y %t1 = zext <3 x i16> %t0 to <3 x i32> %t2 = ashr <3 x i32> %x, %t1 %t3 = trunc <3 x i32> %t2 to <3 x i16> - %t4 = add <3 x i16> %y, + %t4 = add <3 x i16> %y, %t5 = ashr <3 x i16> %t3, %t4 ret <3 x i16> %t5 } diff --git a/llvm/test/Transforms/InstCombine/shift-amount-reassociation-with-truncation-lshr.ll b/llvm/test/Transforms/InstCombine/shift-amount-reassociation-with-truncation-lshr.ll index 214ec88d2e55..c31b6ed3ea2b 100644 --- a/llvm/test/Transforms/InstCombine/shift-amount-reassociation-with-truncation-lshr.ll +++ b/llvm/test/Transforms/InstCombine/shift-amount-reassociation-with-truncation-lshr.ll @@ -42,13 +42,13 @@ define <2 x i16> @t1_vec_splat(<2 x i32> %x, <2 x i16> %y) { ret <2 x i16> %t5 } -define <3 x i16> @t3_vec_nonsplat_undef0(<3 x i32> %x, <3 x i16> %y) { -; CHECK-LABEL: @t3_vec_nonsplat_undef0( -; CHECK-NEXT: [[TMP1:%.*]] = lshr <3 x i32> [[X:%.*]], -; CHECK-NEXT: [[T5:%.*]] = trunc <3 x i32> [[TMP1]] to <3 x i16> +define <3 x i16> @t3_vec_nonsplat_poison0(<3 x i32> %x, <3 x i16> %y) { +; CHECK-LABEL: @t3_vec_nonsplat_poison0( +; CHECK-NEXT: [[TMP1:%.*]] = lshr <3 x i32> [[X:%.*]], +; CHECK-NEXT: [[T5:%.*]] = trunc nuw nsw <3 x i32> [[TMP1]] to <3 x i16> ; CHECK-NEXT: ret <3 x i16> [[T5]] ; - %t0 = sub <3 x i16> , %y + %t0 = sub <3 x i16> , %y %t1 = zext <3 x i16> %t0 to <3 x i32> %t2 = lshr <3 x i32> %x, %t1 %t3 = trunc <3 x i32> %t2 to <3 x i16> @@ -57,32 +57,32 @@ define <3 x i16> @t3_vec_nonsplat_undef0(<3 x i32> %x, <3 x i16> %y) { ret <3 x i16> %t5 } -define <3 x i16> @t4_vec_nonsplat_undef1(<3 x i32> %x, <3 x i16> %y) { -; CHECK-LABEL: @t4_vec_nonsplat_undef1( -; CHECK-NEXT: [[TMP1:%.*]] = lshr <3 x i32> [[X:%.*]], -; CHECK-NEXT: [[T5:%.*]] = trunc <3 x i32> [[TMP1]] to <3 x i16> +define <3 x i16> @t4_vec_nonsplat_poison1(<3 x i32> %x, <3 x i16> %y) { +; CHECK-LABEL: @t4_vec_nonsplat_poison1( +; CHECK-NEXT: [[TMP1:%.*]] = lshr <3 x i32> [[X:%.*]], +; CHECK-NEXT: [[T5:%.*]] = trunc nuw nsw <3 x i32> [[TMP1]] to <3 x i16> ; CHECK-NEXT: ret <3 x i16> [[T5]] ; %t0 = sub <3 x i16> , %y %t1 = zext <3 x i16> %t0 to <3 x i32> %t2 = lshr <3 x i32> %x, %t1 %t3 = trunc <3 x i32> %t2 to <3 x i16> - %t4 = add <3 x i16> %y, + %t4 = add <3 x i16> %y, %t5 = lshr <3 x i16> %t3, %t4 ret <3 x i16> %t5 } -define <3 x i16> @t5_vec_nonsplat_undef1(<3 x i32> %x, <3 x i16> %y) { -; CHECK-LABEL: @t5_vec_nonsplat_undef1( -; CHECK-NEXT: [[TMP1:%.*]] = lshr <3 x i32> [[X:%.*]], -; CHECK-NEXT: [[T5:%.*]] = trunc <3 x i32> [[TMP1]] to <3 x i16> +define <3 x i16> @t5_vec_nonsplat_poison1(<3 x i32> %x, <3 x i16> %y) { +; CHECK-LABEL: @t5_vec_nonsplat_poison1( +; CHECK-NEXT: [[TMP1:%.*]] = lshr <3 x i32> [[X:%.*]], +; CHECK-NEXT: [[T5:%.*]] = trunc nuw nsw <3 x i32> [[TMP1]] to <3 x i16> ; CHECK-NEXT: ret <3 x i16> [[T5]] ; - %t0 = sub <3 x i16> , %y + %t0 = sub <3 x i16> , %y %t1 = zext <3 x i16> %t0 to <3 x i32> %t2 = lshr <3 x i32> %x, %t1 %t3 = trunc <3 x i32> %t2 to <3 x i16> - %t4 = add <3 x i16> %y, + %t4 = add <3 x i16> %y, %t5 = lshr <3 x i16> %t3, %t4 ret <3 x i16> %t5 } diff --git a/llvm/test/Transforms/InstCombine/shift-amount-reassociation.ll b/llvm/test/Transforms/InstCombine/shift-amount-reassociation.ll index b96bcd6bab4f..6bbe4c5151e4 100644 --- a/llvm/test/Transforms/InstCombine/shift-amount-reassociation.ll +++ b/llvm/test/Transforms/InstCombine/shift-amount-reassociation.ll @@ -48,38 +48,38 @@ define <2 x i32> @t2_vec_nonsplat(<2 x i32> %x, <2 x i32> %y) { ; Basic vector tests -define <3 x i32> @t3_vec_nonsplat_undef0(<3 x i32> %x, <3 x i32> %y) { -; CHECK-LABEL: @t3_vec_nonsplat_undef0( -; CHECK-NEXT: [[T3:%.*]] = lshr <3 x i32> [[X:%.*]], +define <3 x i32> @t3_vec_nonsplat_poison0(<3 x i32> %x, <3 x i32> %y) { +; CHECK-LABEL: @t3_vec_nonsplat_poison0( +; CHECK-NEXT: [[T3:%.*]] = lshr <3 x i32> [[X:%.*]], ; CHECK-NEXT: ret <3 x i32> [[T3]] ; - %t0 = sub <3 x i32> , %y + %t0 = sub <3 x i32> , %y %t1 = lshr <3 x i32> %x, %t0 %t2 = add <3 x i32> %y, %t3 = lshr <3 x i32> %t1, %t2 ret <3 x i32> %t3 } -define <3 x i32> @t4_vec_nonsplat_undef1(<3 x i32> %x, <3 x i32> %y) { -; CHECK-LABEL: @t4_vec_nonsplat_undef1( -; CHECK-NEXT: [[T3:%.*]] = lshr <3 x i32> [[X:%.*]], +define <3 x i32> @t4_vec_nonsplat_poison1(<3 x i32> %x, <3 x i32> %y) { +; CHECK-LABEL: @t4_vec_nonsplat_poison1( +; CHECK-NEXT: [[T3:%.*]] = lshr <3 x i32> [[X:%.*]], ; CHECK-NEXT: ret <3 x i32> [[T3]] ; %t0 = sub <3 x i32> , %y %t1 = lshr <3 x i32> %x, %t0 - %t2 = add <3 x i32> %y, + %t2 = add <3 x i32> %y, %t3 = lshr <3 x i32> %t1, %t2 ret <3 x i32> %t3 } -define <3 x i32> @t5_vec_nonsplat_undef1(<3 x i32> %x, <3 x i32> %y) { -; CHECK-LABEL: @t5_vec_nonsplat_undef1( -; CHECK-NEXT: [[T3:%.*]] = lshr <3 x i32> [[X:%.*]], +define <3 x i32> @t5_vec_nonsplat_poison1(<3 x i32> %x, <3 x i32> %y) { +; CHECK-LABEL: @t5_vec_nonsplat_poison1( +; CHECK-NEXT: [[T3:%.*]] = lshr <3 x i32> [[X:%.*]], ; CHECK-NEXT: ret <3 x i32> [[T3]] ; - %t0 = sub <3 x i32> , %y + %t0 = sub <3 x i32> , %y %t1 = lshr <3 x i32> %x, %t0 - %t2 = add <3 x i32> %y, + %t2 = add <3 x i32> %y, %t3 = lshr <3 x i32> %t1, %t2 ret <3 x i32> %t3 } diff --git a/llvm/test/Transforms/InstCombine/shift-logic.ll b/llvm/test/Transforms/InstCombine/shift-logic.ll index c982b45b504e..b591400c6a26 100644 --- a/llvm/test/Transforms/InstCombine/shift-logic.ll +++ b/llvm/test/Transforms/InstCombine/shift-logic.ll @@ -44,18 +44,18 @@ define i16 @shl_or(i16 %x, i16 %py) { ret i16 %sh1 } -define <2 x i16> @shl_or_undef(<2 x i16> %x, <2 x i16> %py) { -; CHECK-LABEL: @shl_or_undef( +define <2 x i16> @shl_or_poison(<2 x i16> %x, <2 x i16> %py) { +; CHECK-LABEL: @shl_or_poison( ; CHECK-NEXT: [[Y:%.*]] = srem <2 x i16> [[PY:%.*]], -; CHECK-NEXT: [[TMP1:%.*]] = shl <2 x i16> [[X:%.*]], -; CHECK-NEXT: [[TMP2:%.*]] = shl <2 x i16> [[Y]], +; CHECK-NEXT: [[TMP1:%.*]] = shl <2 x i16> [[X:%.*]], +; CHECK-NEXT: [[TMP2:%.*]] = shl nsw <2 x i16> [[Y]], ; CHECK-NEXT: [[SH1:%.*]] = or <2 x i16> [[TMP1]], [[TMP2]] ; CHECK-NEXT: ret <2 x i16> [[SH1]] ; %y = srem <2 x i16> %py, ; thwart complexity-based canonicalization - %sh0 = shl <2 x i16> %x, + %sh0 = shl <2 x i16> %x, %r = or <2 x i16> %y, %sh0 - %sh1 = shl <2 x i16> %r, + %sh1 = shl <2 x i16> %r, ret <2 x i16> %sh1 } @@ -100,18 +100,18 @@ define i64 @lshr_and(i64 %x, i64 %py) { ret i64 %sh1 } -define <2 x i64> @lshr_and_undef(<2 x i64> %x, <2 x i64> %py) { -; CHECK-LABEL: @lshr_and_undef( +define <2 x i64> @lshr_and_poison(<2 x i64> %x, <2 x i64> %py) { +; CHECK-LABEL: @lshr_and_poison( ; CHECK-NEXT: [[Y:%.*]] = srem <2 x i64> [[PY:%.*]], -; CHECK-NEXT: [[TMP1:%.*]] = lshr <2 x i64> [[X:%.*]], -; CHECK-NEXT: [[TMP2:%.*]] = lshr <2 x i64> [[Y]], +; CHECK-NEXT: [[TMP1:%.*]] = lshr <2 x i64> [[X:%.*]], +; CHECK-NEXT: [[TMP2:%.*]] = lshr <2 x i64> [[Y]], ; CHECK-NEXT: [[SH1:%.*]] = and <2 x i64> [[TMP1]], [[TMP2]] ; CHECK-NEXT: ret <2 x i64> [[SH1]] ; %y = srem <2 x i64> %py, ; thwart complexity-based canonicalization - %sh0 = lshr <2 x i64> %x, + %sh0 = lshr <2 x i64> %x, %r = and <2 x i64> %y, %sh0 - %sh1 = lshr <2 x i64> %r, + %sh1 = lshr <2 x i64> %r, ret <2 x i64> %sh1 } @@ -212,16 +212,16 @@ define i32 @ashr_overshift_xor(i32 %x, i32 %y) { ret i32 %sh1 } -define <2 x i32> @ashr_undef_undef_xor(<2 x i32> %x, <2 x i32> %y) { -; CHECK-LABEL: @ashr_undef_undef_xor( -; CHECK-NEXT: [[SH0:%.*]] = ashr <2 x i32> [[X:%.*]], +define <2 x i32> @ashr_poison_poison_xor(<2 x i32> %x, <2 x i32> %y) { +; CHECK-LABEL: @ashr_poison_poison_xor( +; CHECK-NEXT: [[SH0:%.*]] = ashr <2 x i32> [[X:%.*]], ; CHECK-NEXT: [[R:%.*]] = xor <2 x i32> [[SH0]], [[Y:%.*]] -; CHECK-NEXT: [[SH1:%.*]] = ashr <2 x i32> [[R]], +; CHECK-NEXT: [[SH1:%.*]] = ashr <2 x i32> [[R]], ; CHECK-NEXT: ret <2 x i32> [[SH1]] ; - %sh0 = ashr <2 x i32> %x, + %sh0 = ashr <2 x i32> %x, %r = xor <2 x i32> %y, %sh0 - %sh1 = ashr <2 x i32> %r, + %sh1 = ashr <2 x i32> %r, ret <2 x i32> %sh1 } @@ -390,18 +390,18 @@ define <2 x i8> @shl_add_nonuniform(<2 x i8> %x, <2 x i8> %y) { } -define <2 x i64> @shl_add_undef(<2 x i64> %x, <2 x i64> %py) { -; CHECK-LABEL: @shl_add_undef( +define <2 x i64> @shl_add_poison(<2 x i64> %x, <2 x i64> %py) { +; CHECK-LABEL: @shl_add_poison( ; CHECK-NEXT: [[Y:%.*]] = srem <2 x i64> [[PY:%.*]], -; CHECK-NEXT: [[TMP1:%.*]] = shl <2 x i64> [[X:%.*]], -; CHECK-NEXT: [[TMP2:%.*]] = shl <2 x i64> [[Y]], +; CHECK-NEXT: [[TMP1:%.*]] = shl <2 x i64> [[X:%.*]], +; CHECK-NEXT: [[TMP2:%.*]] = shl nsw <2 x i64> [[Y]], ; CHECK-NEXT: [[SH1:%.*]] = add <2 x i64> [[TMP1]], [[TMP2]] ; CHECK-NEXT: ret <2 x i64> [[SH1]] ; %y = srem <2 x i64> %py, ; thwart complexity-based canonicalization - %sh0 = shl <2 x i64> %x, + %sh0 = shl <2 x i64> %x, %r = add <2 x i64> %y, %sh0 - %sh1 = shl <2 x i64> %r, + %sh1 = shl <2 x i64> %r, ret <2 x i64> %sh1 } @@ -432,18 +432,18 @@ define <2 x i8> @lshr_add_nonuniform(<2 x i8> %x, <2 x i8> %y) { ret <2 x i8> %sh1 } -define <2 x i64> @lshr_add_undef(<2 x i64> %x, <2 x i64> %py) { -; CHECK-LABEL: @lshr_add_undef( +define <2 x i64> @lshr_add_poison(<2 x i64> %x, <2 x i64> %py) { +; CHECK-LABEL: @lshr_add_poison( ; CHECK-NEXT: [[Y:%.*]] = srem <2 x i64> [[PY:%.*]], -; CHECK-NEXT: [[SH0:%.*]] = lshr <2 x i64> [[X:%.*]], -; CHECK-NEXT: [[R:%.*]] = add <2 x i64> [[Y]], [[SH0]] -; CHECK-NEXT: [[SH1:%.*]] = lshr <2 x i64> [[R]], +; CHECK-NEXT: [[SH0:%.*]] = lshr <2 x i64> [[X:%.*]], +; CHECK-NEXT: [[R:%.*]] = add nsw <2 x i64> [[Y]], [[SH0]] +; CHECK-NEXT: [[SH1:%.*]] = lshr <2 x i64> [[R]], ; CHECK-NEXT: ret <2 x i64> [[SH1]] ; %y = srem <2 x i64> %py, ; thwart complexity-based canonicalization - %sh0 = lshr <2 x i64> %x, + %sh0 = lshr <2 x i64> %x, %r = add <2 x i64> %y, %sh0 - %sh1 = lshr <2 x i64> %r, + %sh1 = lshr <2 x i64> %r, ret <2 x i64> %sh1 } @@ -488,18 +488,18 @@ define <2 x i8> @shl_sub_nonuniform(<2 x i8> %x, <2 x i8> %y) { } -define <2 x i64> @shl_sub_undef(<2 x i64> %x, <2 x i64> %py) { -; CHECK-LABEL: @shl_sub_undef( +define <2 x i64> @shl_sub_poison(<2 x i64> %x, <2 x i64> %py) { +; CHECK-LABEL: @shl_sub_poison( ; CHECK-NEXT: [[Y:%.*]] = srem <2 x i64> [[PY:%.*]], -; CHECK-NEXT: [[TMP1:%.*]] = shl <2 x i64> [[X:%.*]], -; CHECK-NEXT: [[TMP2:%.*]] = shl <2 x i64> [[Y]], +; CHECK-NEXT: [[TMP1:%.*]] = shl <2 x i64> [[X:%.*]], +; CHECK-NEXT: [[TMP2:%.*]] = shl nsw <2 x i64> [[Y]], ; CHECK-NEXT: [[SH1:%.*]] = sub <2 x i64> [[TMP2]], [[TMP1]] ; CHECK-NEXT: ret <2 x i64> [[SH1]] ; %y = srem <2 x i64> %py, ; thwart complexity-based canonicalization - %sh0 = shl <2 x i64> %x, + %sh0 = shl <2 x i64> %x, %r = sub <2 x i64> %y, %sh0 - %sh1 = shl <2 x i64> %r, + %sh1 = shl <2 x i64> %r, ret <2 x i64> %sh1 } @@ -530,17 +530,17 @@ define <2 x i8> @lshr_sub_nonuniform(<2 x i8> %x, <2 x i8> %y) { ret <2 x i8> %sh1 } -define <2 x i64> @lshr_sub_undef(<2 x i64> %x, <2 x i64> %py) { -; CHECK-LABEL: @lshr_sub_undef( +define <2 x i64> @lshr_sub_poison(<2 x i64> %x, <2 x i64> %py) { +; CHECK-LABEL: @lshr_sub_poison( ; CHECK-NEXT: [[Y:%.*]] = srem <2 x i64> [[PY:%.*]], -; CHECK-NEXT: [[SH0:%.*]] = lshr <2 x i64> [[X:%.*]], -; CHECK-NEXT: [[R:%.*]] = sub <2 x i64> [[Y]], [[SH0]] -; CHECK-NEXT: [[SH1:%.*]] = lshr <2 x i64> [[R]], +; CHECK-NEXT: [[SH0:%.*]] = lshr <2 x i64> [[X:%.*]], +; CHECK-NEXT: [[R:%.*]] = sub nsw <2 x i64> [[Y]], [[SH0]] +; CHECK-NEXT: [[SH1:%.*]] = lshr <2 x i64> [[R]], ; CHECK-NEXT: ret <2 x i64> [[SH1]] ; %y = srem <2 x i64> %py, ; thwart complexity-based canonicalization - %sh0 = lshr <2 x i64> %x, + %sh0 = lshr <2 x i64> %x, %r = sub <2 x i64> %y, %sh0 - %sh1 = lshr <2 x i64> %r, + %sh1 = lshr <2 x i64> %r, ret <2 x i64> %sh1 } diff --git a/llvm/test/Transforms/InstCombine/shl-and-negC-icmpeq-zero.ll b/llvm/test/Transforms/InstCombine/shl-and-negC-icmpeq-zero.ll index 406dc72f2646..daa495579659 100644 --- a/llvm/test/Transforms/InstCombine/shl-and-negC-icmpeq-zero.ll +++ b/llvm/test/Transforms/InstCombine/shl-and-negC-icmpeq-zero.ll @@ -81,39 +81,39 @@ define <4 x i1> @vec_4xi32_shl_and_negC_eq(<4 x i32> %x, <4 x i32> %y) { ret <4 x i1> %r } -define <4 x i1> @vec_shl_and_negC_eq_undef1(<4 x i32> %x, <4 x i32> %y) { -; CHECK-LABEL: @vec_shl_and_negC_eq_undef1( +define <4 x i1> @vec_shl_and_negC_eq_poison1(<4 x i32> %x, <4 x i32> %y) { +; CHECK-LABEL: @vec_shl_and_negC_eq_poison1( ; CHECK-NEXT: [[SHL:%.*]] = shl <4 x i32> [[X:%.*]], [[Y:%.*]] ; CHECK-NEXT: [[R:%.*]] = icmp ult <4 x i32> [[SHL]], ; CHECK-NEXT: ret <4 x i1> [[R]] ; %shl = shl <4 x i32> %x, %y - %and = and <4 x i32> %shl, ; ~7 + %and = and <4 x i32> %shl, ; ~7 %r = icmp eq <4 x i32> %and, ret <4 x i1> %r } -define <4 x i1> @vec_shl_and_negC_eq_undef2(<4 x i32> %x, <4 x i32> %y) { -; CHECK-LABEL: @vec_shl_and_negC_eq_undef2( +define <4 x i1> @vec_shl_and_negC_eq_poison2(<4 x i32> %x, <4 x i32> %y) { +; CHECK-LABEL: @vec_shl_and_negC_eq_poison2( ; CHECK-NEXT: [[SHL:%.*]] = shl <4 x i32> [[X:%.*]], [[Y:%.*]] ; CHECK-NEXT: [[R:%.*]] = icmp ult <4 x i32> [[SHL]], ; CHECK-NEXT: ret <4 x i1> [[R]] ; %shl = shl <4 x i32> %x, %y %and = and <4 x i32> %shl, ; ~7 - %r = icmp eq <4 x i32> %and, + %r = icmp eq <4 x i32> %and, ret <4 x i1> %r } -define <4 x i1> @vec_shl_and_negC_eq_undef3(<4 x i32> %x, <4 x i32> %y) { -; CHECK-LABEL: @vec_shl_and_negC_eq_undef3( +define <4 x i1> @vec_shl_and_negC_eq_poison3(<4 x i32> %x, <4 x i32> %y) { +; CHECK-LABEL: @vec_shl_and_negC_eq_poison3( ; CHECK-NEXT: [[SHL:%.*]] = shl <4 x i32> [[X:%.*]], [[Y:%.*]] ; CHECK-NEXT: [[R:%.*]] = icmp ult <4 x i32> [[SHL]], ; CHECK-NEXT: ret <4 x i1> [[R]] ; %shl = shl <4 x i32> %x, %y - %and = and <4 x i32> %shl, ; ~7 - %r = icmp eq <4 x i32> %and, + %and = and <4 x i32> %shl, ; ~7 + %r = icmp eq <4 x i32> %and, ret <4 x i1> %r } diff --git a/llvm/test/Transforms/InstCombine/shl-and-signbit-icmpeq-zero.ll b/llvm/test/Transforms/InstCombine/shl-and-signbit-icmpeq-zero.ll index 4c2c876e3925..dcc181945357 100644 --- a/llvm/test/Transforms/InstCombine/shl-and-signbit-icmpeq-zero.ll +++ b/llvm/test/Transforms/InstCombine/shl-and-signbit-icmpeq-zero.ll @@ -81,39 +81,39 @@ define <4 x i1> @vec_4xi32_shl_and_signbit_eq(<4 x i32> %x, <4 x i32> %y) { ret <4 x i1> %r } -define <4 x i1> @vec_4xi32_shl_and_signbit_eq_undef1(<4 x i32> %x, <4 x i32> %y) { -; CHECK-LABEL: @vec_4xi32_shl_and_signbit_eq_undef1( +define <4 x i1> @vec_4xi32_shl_and_signbit_eq_poison1(<4 x i32> %x, <4 x i32> %y) { +; CHECK-LABEL: @vec_4xi32_shl_and_signbit_eq_poison1( ; CHECK-NEXT: [[SHL:%.*]] = shl <4 x i32> [[X:%.*]], [[Y:%.*]] ; CHECK-NEXT: [[R:%.*]] = icmp sgt <4 x i32> [[SHL]], ; CHECK-NEXT: ret <4 x i1> [[R]] ; %shl = shl <4 x i32> %x, %y - %and = and <4 x i32> %shl, + %and = and <4 x i32> %shl, %r = icmp eq <4 x i32> %and, ret <4 x i1> %r } -define <4 x i1> @vec_4xi32_shl_and_signbit_eq_undef2(<4 x i32> %x, <4 x i32> %y) { -; CHECK-LABEL: @vec_4xi32_shl_and_signbit_eq_undef2( +define <4 x i1> @vec_4xi32_shl_and_signbit_eq_poison2(<4 x i32> %x, <4 x i32> %y) { +; CHECK-LABEL: @vec_4xi32_shl_and_signbit_eq_poison2( ; CHECK-NEXT: [[SHL:%.*]] = shl <4 x i32> [[X:%.*]], [[Y:%.*]] ; CHECK-NEXT: [[R:%.*]] = icmp sgt <4 x i32> [[SHL]], ; CHECK-NEXT: ret <4 x i1> [[R]] ; %shl = shl <4 x i32> %x, %y %and = and <4 x i32> %shl, - %r = icmp eq <4 x i32> %and, + %r = icmp eq <4 x i32> %and, ret <4 x i1> %r } -define <4 x i1> @vec_4xi32_shl_and_signbit_eq_undef3(<4 x i32> %x, <4 x i32> %y) { -; CHECK-LABEL: @vec_4xi32_shl_and_signbit_eq_undef3( +define <4 x i1> @vec_4xi32_shl_and_signbit_eq_poison3(<4 x i32> %x, <4 x i32> %y) { +; CHECK-LABEL: @vec_4xi32_shl_and_signbit_eq_poison3( ; CHECK-NEXT: [[SHL:%.*]] = shl <4 x i32> [[X:%.*]], [[Y:%.*]] ; CHECK-NEXT: [[R:%.*]] = icmp sgt <4 x i32> [[SHL]], ; CHECK-NEXT: ret <4 x i1> [[R]] ; %shl = shl <4 x i32> %x, %y - %and = and <4 x i32> %shl, - %r = icmp eq <4 x i32> %and, + %and = and <4 x i32> %shl, + %r = icmp eq <4 x i32> %and, ret <4 x i1> %r } diff --git a/llvm/test/Transforms/InstCombine/signmask-of-sext-vs-of-shl-of-zext.ll b/llvm/test/Transforms/InstCombine/signmask-of-sext-vs-of-shl-of-zext.ll index aeb4c8bb62cb..e7505721cad6 100644 --- a/llvm/test/Transforms/InstCombine/signmask-of-sext-vs-of-shl-of-zext.ll +++ b/llvm/test/Transforms/InstCombine/signmask-of-sext-vs-of-shl-of-zext.ll @@ -129,40 +129,56 @@ define <2 x i32> @t8(<2 x i16> %x) { %r = and <2 x i32> %i1, ret <2 x i32> %r } + define <2 x i32> @t9(<2 x i16> %x) { ; CHECK-LABEL: @t9( -; CHECK-NEXT: [[X_SIGNEXT:%.*]] = sext <2 x i16> [[X:%.*]] to <2 x i32> -; CHECK-NEXT: [[R:%.*]] = and <2 x i32> [[X_SIGNEXT]], +; CHECK-NEXT: [[I1:%.*]] = sext <2 x i16> [[X:%.*]] to <2 x i32> +; CHECK-NEXT: [[R:%.*]] = and <2 x i32> [[I1]], ; CHECK-NEXT: ret <2 x i32> [[R]] ; %i0 = zext <2 x i16> %x to <2 x i32> - %i1 = shl <2 x i32> %i0, + %i1 = shl <2 x i32> %i0, %r = and <2 x i32> %i1, - ; Here undef can be propagated into the mask. ret <2 x i32> %r } -define <2 x i32> @t10(<2 x i16> %x) { -; CHECK-LABEL: @t10( -; CHECK-NEXT: [[X_SIGNEXT:%.*]] = sext <2 x i16> [[X:%.*]] to <2 x i32> -; CHECK-NEXT: [[R:%.*]] = and <2 x i32> [[X_SIGNEXT]], + +; If we folded this, we wouldn't be able to keep the undef mask. +define <2 x i32> @t10_undef(<2 x i16> %x) { +; CHECK-LABEL: @t10_undef( +; CHECK-NEXT: [[I0:%.*]] = zext <2 x i16> [[X:%.*]] to <2 x i32> +; CHECK-NEXT: [[I1:%.*]] = shl nuw <2 x i32> [[I0]], +; CHECK-NEXT: [[R:%.*]] = and <2 x i32> [[I1]], ; CHECK-NEXT: ret <2 x i32> [[R]] ; %i0 = zext <2 x i16> %x to <2 x i32> %i1 = shl <2 x i32> %i0, %r = and <2 x i32> %i1, - ; CAREFUL! We can't keep undef mask here, since high bits are no longer zero, + ret <2 x i32> %r +} + +define <2 x i32> @t10_poison(<2 x i16> %x) { +; CHECK-LABEL: @t10_poison( +; CHECK-NEXT: [[I1:%.*]] = sext <2 x i16> [[X:%.*]] to <2 x i32> +; CHECK-NEXT: [[R:%.*]] = and <2 x i32> [[I1]], +; CHECK-NEXT: ret <2 x i32> [[R]] +; + %i0 = zext <2 x i16> %x to <2 x i32> + %i1 = shl <2 x i32> %i0, + %r = and <2 x i32> %i1, + ; CAREFUL! We can't keep poison mask here, since high bits are no longer zero, ; we must sanitize it to 0. ret <2 x i32> %r } + define <2 x i32> @t11(<2 x i16> %x) { ; CHECK-LABEL: @t11( ; CHECK-NEXT: [[X_SIGNEXT:%.*]] = sext <2 x i16> [[X:%.*]] to <2 x i32> -; CHECK-NEXT: [[R:%.*]] = and <2 x i32> [[X_SIGNEXT]], +; CHECK-NEXT: [[R:%.*]] = and <2 x i32> [[X_SIGNEXT]], ; CHECK-NEXT: ret <2 x i32> [[R]] ; %i0 = zext <2 x i16> %x to <2 x i32> - %i1 = shl <2 x i32> %i0, - %r = and <2 x i32> %i1, - ; Here undef mask is fine. + %i1 = shl <2 x i32> %i0, + %r = and <2 x i32> %i1, + ; Here poison mask is fine. ret <2 x i32> %r } diff --git a/llvm/test/Transforms/InstCombine/sub-not.ll b/llvm/test/Transforms/InstCombine/sub-not.ll index ec36754d3e9b..89ccf5aa3c8f 100644 --- a/llvm/test/Transforms/InstCombine/sub-not.ll +++ b/llvm/test/Transforms/InstCombine/sub-not.ll @@ -34,7 +34,7 @@ define <2 x i8> @sub_not_vec(<2 x i8> %x, <2 x i8> %y) { ; CHECK-NEXT: ret <2 x i8> [[R]] ; %s = sub <2 x i8> %x, %y - %r = xor <2 x i8> %s, + %r = xor <2 x i8> %s, ret <2 x i8> %r } @@ -69,7 +69,7 @@ define <2 x i8> @dec_sub_vec(<2 x i8> %x, <2 x i8> %y) { ; CHECK-NEXT: ret <2 x i8> [[R]] ; %s = sub <2 x i8> %x, %y - %r = add <2 x i8> %s, + %r = add <2 x i8> %s, ret <2 x i8> %r } @@ -103,7 +103,7 @@ define <2 x i8> @sub_inc_vec(<2 x i8> %x, <2 x i8> %y) { ; CHECK-NEXT: [[R:%.*]] = add <2 x i8> [[S_NEG]], [[Y:%.*]] ; CHECK-NEXT: ret <2 x i8> [[R]] ; - %s = add <2 x i8> %x, + %s = add <2 x i8> %x, %r = sub <2 x i8> %y, %s ret <2 x i8> %r } @@ -138,7 +138,7 @@ define <2 x i8> @sub_dec_vec(<2 x i8> %x, <2 x i8> %y) { ; CHECK-NEXT: [[R:%.*]] = add <2 x i8> [[TMP1]], [[X:%.*]] ; CHECK-NEXT: ret <2 x i8> [[R]] ; - %s = add <2 x i8> %x, + %s = add <2 x i8> %x, %r = sub <2 x i8> %s, %y ret <2 x i8> %r } diff --git a/llvm/test/Transforms/InstCombine/sub.ll b/llvm/test/Transforms/InstCombine/sub.ll index 249b5673c8ac..a84e389f13c3 100644 --- a/llvm/test/Transforms/InstCombine/sub.ll +++ b/llvm/test/Transforms/InstCombine/sub.ll @@ -130,44 +130,44 @@ define <2 x i32> @neg_nsw_sub_nsw_vec(<2 x i32> %x, <2 x i32> %y) { ret <2 x i32> %r } -define <2 x i32> @neg_sub_vec_undef(<2 x i32> %x, <2 x i32> %y) { -; CHECK-LABEL: @neg_sub_vec_undef( +define <2 x i32> @neg_sub_vec_poison(<2 x i32> %x, <2 x i32> %y) { +; CHECK-LABEL: @neg_sub_vec_poison( ; CHECK-NEXT: [[R:%.*]] = add <2 x i32> [[Y:%.*]], [[X:%.*]] ; CHECK-NEXT: ret <2 x i32> [[R]] ; - %neg = sub <2 x i32> , %x + %neg = sub <2 x i32> , %x %r = sub <2 x i32> %y, %neg ret <2 x i32> %r } -define <2 x i32> @neg_nsw_sub_vec_undef(<2 x i32> %x, <2 x i32> %y) { -; CHECK-LABEL: @neg_nsw_sub_vec_undef( +define <2 x i32> @neg_nsw_sub_vec_poison(<2 x i32> %x, <2 x i32> %y) { +; CHECK-LABEL: @neg_nsw_sub_vec_poison( ; CHECK-NEXT: [[R:%.*]] = add <2 x i32> [[Y:%.*]], [[X:%.*]] ; CHECK-NEXT: ret <2 x i32> [[R]] ; - %neg = sub nsw <2 x i32> , %x + %neg = sub nsw <2 x i32> , %x %r = sub <2 x i32> %y, %neg ret <2 x i32> %r } -define <2 x i32> @neg_sub_nsw_vec_undef(<2 x i32> %x, <2 x i32> %y) { -; CHECK-LABEL: @neg_sub_nsw_vec_undef( +define <2 x i32> @neg_sub_nsw_vec_poison(<2 x i32> %x, <2 x i32> %y) { +; CHECK-LABEL: @neg_sub_nsw_vec_poison( ; CHECK-NEXT: [[R:%.*]] = add <2 x i32> [[Y:%.*]], [[X:%.*]] ; CHECK-NEXT: ret <2 x i32> [[R]] ; - %neg = sub <2 x i32> , %x + %neg = sub <2 x i32> , %x %r = sub nsw <2 x i32> %y, %neg ret <2 x i32> %r } ; This should not drop 'nsw'. -define <2 x i32> @neg_nsw_sub_nsw_vec_undef(<2 x i32> %x, <2 x i32> %y) { -; CHECK-LABEL: @neg_nsw_sub_nsw_vec_undef( +define <2 x i32> @neg_nsw_sub_nsw_vec_poison(<2 x i32> %x, <2 x i32> %y) { +; CHECK-LABEL: @neg_nsw_sub_nsw_vec_poison( ; CHECK-NEXT: [[R:%.*]] = add nsw <2 x i32> [[Y:%.*]], [[X:%.*]] ; CHECK-NEXT: ret <2 x i32> [[R]] ; - %neg = sub nsw <2 x i32> , %x + %neg = sub nsw <2 x i32> , %x %r = sub nsw <2 x i32> %y, %neg ret <2 x i32> %r } @@ -205,13 +205,13 @@ define <2 x i8> @notnotsub_vec(<2 x i8> %x, <2 x i8> %y) { ret <2 x i8> %sub } -define <2 x i8> @notnotsub_vec_undef_elts(<2 x i8> %x, <2 x i8> %y) { -; CHECK-LABEL: @notnotsub_vec_undef_elts( +define <2 x i8> @notnotsub_vec_poison_elts(<2 x i8> %x, <2 x i8> %y) { +; CHECK-LABEL: @notnotsub_vec_poison_elts( ; CHECK-NEXT: [[SUB:%.*]] = sub <2 x i8> [[Y:%.*]], [[X:%.*]] ; CHECK-NEXT: ret <2 x i8> [[SUB]] ; - %nx = xor <2 x i8> %x, - %ny = xor <2 x i8> %y, + %nx = xor <2 x i8> %x, + %ny = xor <2 x i8> %y, %sub = sub <2 x i8> %nx, %ny ret <2 x i8> %sub } @@ -2351,12 +2351,12 @@ define <2 x i8> @sub_to_and_vector1(<2 x i8> %x) { define <2 x i8> @sub_to_and_vector2(<2 x i8> %x) { ; CHECK-LABEL: @sub_to_and_vector2( -; CHECK-NEXT: [[SUB:%.*]] = sub nuw <2 x i8> , [[X:%.*]] +; CHECK-NEXT: [[SUB:%.*]] = sub nuw <2 x i8> , [[X:%.*]] ; CHECK-NEXT: [[AND:%.*]] = and <2 x i8> [[SUB]], ; CHECK-NEXT: [[R:%.*]] = sub nsw <2 x i8> , [[AND]] ; CHECK-NEXT: ret <2 x i8> [[R]] ; - %sub = sub nuw <2 x i8> , %x + %sub = sub nuw <2 x i8> , %x %and = and <2 x i8> %sub, %r = sub <2 x i8> , %and ret <2 x i8> %r @@ -2366,12 +2366,12 @@ define <2 x i8> @sub_to_and_vector2(<2 x i8> %x) { define <2 x i8> @sub_to_and_vector3(<2 x i8> %x) { ; CHECK-LABEL: @sub_to_and_vector3( ; CHECK-NEXT: [[SUB:%.*]] = sub nuw <2 x i8> , [[X:%.*]] -; CHECK-NEXT: [[AND:%.*]] = and <2 x i8> [[SUB]], +; CHECK-NEXT: [[AND:%.*]] = and <2 x i8> [[SUB]], ; CHECK-NEXT: [[R:%.*]] = sub nsw <2 x i8> , [[AND]] ; CHECK-NEXT: ret <2 x i8> [[R]] ; %sub = sub nuw <2 x i8> , %x - %and = and <2 x i8> %sub, + %and = and <2 x i8> %sub, %r = sub <2 x i8> , %and ret <2 x i8> %r } @@ -2381,12 +2381,12 @@ define <2 x i8> @sub_to_and_vector4(<2 x i8> %x) { ; CHECK-LABEL: @sub_to_and_vector4( ; CHECK-NEXT: [[SUB:%.*]] = sub nuw <2 x i8> , [[X:%.*]] ; CHECK-NEXT: [[AND:%.*]] = and <2 x i8> [[SUB]], -; CHECK-NEXT: [[R:%.*]] = sub <2 x i8> , [[AND]] +; CHECK-NEXT: [[R:%.*]] = sub nsw <2 x i8> , [[AND]] ; CHECK-NEXT: ret <2 x i8> [[R]] ; %sub = sub nuw <2 x i8> , %x %and = and <2 x i8> %sub, - %r = sub <2 x i8> , %and + %r = sub <2 x i8> , %and ret <2 x i8> %r } diff --git a/llvm/test/Transforms/InstCombine/trunc-inseltpoison.ll b/llvm/test/Transforms/InstCombine/trunc-inseltpoison.ll index 4c857125365a..063006ba5eea 100644 --- a/llvm/test/Transforms/InstCombine/trunc-inseltpoison.ll +++ b/llvm/test/Transforms/InstCombine/trunc-inseltpoison.ll @@ -49,15 +49,15 @@ define <2 x i64> @test1_vec_nonuniform(<2 x i64> %a) { ret <2 x i64> %d } -define <2 x i64> @test1_vec_undef(<2 x i64> %a) { -; CHECK-LABEL: @test1_vec_undef( +define <2 x i64> @test1_vec_poison(<2 x i64> %a) { +; CHECK-LABEL: @test1_vec_poison( ; CHECK-NEXT: [[B:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i32> -; CHECK-NEXT: [[D:%.*]] = and <2 x i64> [[A]], +; CHECK-NEXT: [[D:%.*]] = and <2 x i64> [[A]], ; CHECK-NEXT: call void @use_vec(<2 x i32> [[B]]) ; CHECK-NEXT: ret <2 x i64> [[D]] ; %b = trunc <2 x i64> %a to <2 x i32> - %c = and <2 x i32> %b, + %c = and <2 x i32> %b, %d = zext <2 x i32> %c to <2 x i64> call void @use_vec(<2 x i32> %b) ret <2 x i64> %d @@ -111,17 +111,17 @@ define <2 x i64> @test2_vec_nonuniform(<2 x i64> %a) { ret <2 x i64> %d } -define <2 x i64> @test2_vec_undef(<2 x i64> %a) { -; CHECK-LABEL: @test2_vec_undef( +define <2 x i64> @test2_vec_poison(<2 x i64> %a) { +; CHECK-LABEL: @test2_vec_poison( ; CHECK-NEXT: [[B:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i32> -; CHECK-NEXT: [[D1:%.*]] = shl <2 x i64> [[A]], -; CHECK-NEXT: [[D:%.*]] = ashr exact <2 x i64> [[D1]], +; CHECK-NEXT: [[D1:%.*]] = shl <2 x i64> [[A]], +; CHECK-NEXT: [[D:%.*]] = ashr exact <2 x i64> [[D1]], ; CHECK-NEXT: call void @use_vec(<2 x i32> [[B]]) ; CHECK-NEXT: ret <2 x i64> [[D]] ; %b = trunc <2 x i64> %a to <2 x i32> - %c = shl <2 x i32> %b, - %q = ashr <2 x i32> %c, + %c = shl <2 x i32> %b, + %q = ashr <2 x i32> %c, %d = sext <2 x i32> %q to <2 x i64> call void @use_vec(<2 x i32> %b) ret <2 x i64> %d @@ -300,18 +300,17 @@ define <2 x i64> @test8_vec_nonuniform(<2 x i32> %A, <2 x i32> %B) { ret <2 x i64> %G } -define <2 x i64> @test8_vec_undef(<2 x i32> %A, <2 x i32> %B) { -; CHECK-LABEL: @test8_vec_undef( -; CHECK-NEXT: [[C:%.*]] = zext <2 x i32> [[A:%.*]] to <2 x i128> -; CHECK-NEXT: [[D:%.*]] = zext <2 x i32> [[B:%.*]] to <2 x i128> -; CHECK-NEXT: [[E:%.*]] = shl <2 x i128> [[D]], -; CHECK-NEXT: [[F:%.*]] = or <2 x i128> [[E]], [[C]] -; CHECK-NEXT: [[G:%.*]] = trunc <2 x i128> [[F]] to <2 x i64> +define <2 x i64> @test8_vec_poison(<2 x i32> %A, <2 x i32> %B) { +; CHECK-LABEL: @test8_vec_poison( +; CHECK-NEXT: [[C:%.*]] = zext <2 x i32> [[A:%.*]] to <2 x i64> +; CHECK-NEXT: [[D:%.*]] = zext <2 x i32> [[B:%.*]] to <2 x i64> +; CHECK-NEXT: [[E:%.*]] = shl nuw <2 x i64> [[D]], +; CHECK-NEXT: [[G:%.*]] = or disjoint <2 x i64> [[E]], [[C]] ; CHECK-NEXT: ret <2 x i64> [[G]] ; %C = zext <2 x i32> %A to <2 x i128> %D = zext <2 x i32> %B to <2 x i128> - %E = shl <2 x i128> %D, + %E = shl <2 x i128> %D, %F = or <2 x i128> %E, %C %G = trunc <2 x i128> %F to <2 x i64> ret <2 x i64> %G @@ -388,18 +387,17 @@ define <2 x i64> @test11_vec_nonuniform(<2 x i32> %A, <2 x i32> %B) { ret <2 x i64> %G } -define <2 x i64> @test11_vec_undef(<2 x i32> %A, <2 x i32> %B) { -; CHECK-LABEL: @test11_vec_undef( -; CHECK-NEXT: [[C:%.*]] = zext <2 x i32> [[A:%.*]] to <2 x i128> -; CHECK-NEXT: [[D:%.*]] = zext <2 x i32> [[B:%.*]] to <2 x i128> -; CHECK-NEXT: [[E:%.*]] = and <2 x i128> [[D]], -; CHECK-NEXT: [[F:%.*]] = shl <2 x i128> [[C]], [[E]] -; CHECK-NEXT: [[G:%.*]] = trunc <2 x i128> [[F]] to <2 x i64> +define <2 x i64> @test11_vec_poison(<2 x i32> %A, <2 x i32> %B) { +; CHECK-LABEL: @test11_vec_poison( +; CHECK-NEXT: [[C:%.*]] = zext <2 x i32> [[A:%.*]] to <2 x i64> +; CHECK-NEXT: [[TMP1:%.*]] = and <2 x i32> [[B:%.*]], +; CHECK-NEXT: [[E:%.*]] = zext nneg <2 x i32> [[TMP1]] to <2 x i64> +; CHECK-NEXT: [[G:%.*]] = shl nuw nsw <2 x i64> [[C]], [[E]] ; CHECK-NEXT: ret <2 x i64> [[G]] ; %C = zext <2 x i32> %A to <2 x i128> %D = zext <2 x i32> %B to <2 x i128> - %E = and <2 x i128> %D, + %E = and <2 x i128> %D, %F = shl <2 x i128> %C, %E %G = trunc <2 x i128> %F to <2 x i64> ret <2 x i64> %G @@ -453,18 +451,17 @@ define <2 x i64> @test12_vec_nonuniform(<2 x i32> %A, <2 x i32> %B) { ret <2 x i64> %G } -define <2 x i64> @test12_vec_undef(<2 x i32> %A, <2 x i32> %B) { -; CHECK-LABEL: @test12_vec_undef( -; CHECK-NEXT: [[C:%.*]] = zext <2 x i32> [[A:%.*]] to <2 x i128> -; CHECK-NEXT: [[D:%.*]] = zext <2 x i32> [[B:%.*]] to <2 x i128> -; CHECK-NEXT: [[E:%.*]] = and <2 x i128> [[D]], -; CHECK-NEXT: [[F:%.*]] = lshr <2 x i128> [[C]], [[E]] -; CHECK-NEXT: [[G:%.*]] = trunc nuw nsw <2 x i128> [[F]] to <2 x i64> +define <2 x i64> @test12_vec_poison(<2 x i32> %A, <2 x i32> %B) { +; CHECK-LABEL: @test12_vec_poison( +; CHECK-NEXT: [[C:%.*]] = zext <2 x i32> [[A:%.*]] to <2 x i64> +; CHECK-NEXT: [[TMP1:%.*]] = and <2 x i32> [[B:%.*]], +; CHECK-NEXT: [[E:%.*]] = zext nneg <2 x i32> [[TMP1]] to <2 x i64> +; CHECK-NEXT: [[G:%.*]] = lshr <2 x i64> [[C]], [[E]] ; CHECK-NEXT: ret <2 x i64> [[G]] ; %C = zext <2 x i32> %A to <2 x i128> %D = zext <2 x i32> %B to <2 x i128> - %E = and <2 x i128> %D, + %E = and <2 x i128> %D, %F = lshr <2 x i128> %C, %E %G = trunc <2 x i128> %F to <2 x i64> ret <2 x i64> %G @@ -518,18 +515,17 @@ define <2 x i64> @test13_vec_nonuniform(<2 x i32> %A, <2 x i32> %B) { ret <2 x i64> %G } -define <2 x i64> @test13_vec_undef(<2 x i32> %A, <2 x i32> %B) { -; CHECK-LABEL: @test13_vec_undef( -; CHECK-NEXT: [[C:%.*]] = sext <2 x i32> [[A:%.*]] to <2 x i128> -; CHECK-NEXT: [[D:%.*]] = zext <2 x i32> [[B:%.*]] to <2 x i128> -; CHECK-NEXT: [[E:%.*]] = and <2 x i128> [[D]], -; CHECK-NEXT: [[F:%.*]] = ashr <2 x i128> [[C]], [[E]] -; CHECK-NEXT: [[G:%.*]] = trunc nsw <2 x i128> [[F]] to <2 x i64> +define <2 x i64> @test13_vec_poison(<2 x i32> %A, <2 x i32> %B) { +; CHECK-LABEL: @test13_vec_poison( +; CHECK-NEXT: [[C:%.*]] = sext <2 x i32> [[A:%.*]] to <2 x i64> +; CHECK-NEXT: [[TMP1:%.*]] = and <2 x i32> [[B:%.*]], +; CHECK-NEXT: [[E:%.*]] = zext nneg <2 x i32> [[TMP1]] to <2 x i64> +; CHECK-NEXT: [[G:%.*]] = ashr <2 x i64> [[C]], [[E]] ; CHECK-NEXT: ret <2 x i64> [[G]] ; %C = sext <2 x i32> %A to <2 x i128> %D = zext <2 x i32> %B to <2 x i128> - %E = and <2 x i128> %D, + %E = and <2 x i128> %D, %F = ashr <2 x i128> %C, %E %G = trunc <2 x i128> %F to <2 x i64> ret <2 x i64> %G @@ -766,13 +762,13 @@ define <2 x i32> @trunc_shl_v2i32_v2i64_uniform(<2 x i64> %val) { ret <2 x i32> %trunc } -define <2 x i32> @trunc_shl_v2i32_v2i64_undef(<2 x i64> %val) { -; CHECK-LABEL: @trunc_shl_v2i32_v2i64_undef( +define <2 x i32> @trunc_shl_v2i32_v2i64_poison(<2 x i64> %val) { +; CHECK-LABEL: @trunc_shl_v2i32_v2i64_poison( ; CHECK-NEXT: [[VAL_TR:%.*]] = trunc <2 x i64> [[VAL:%.*]] to <2 x i32> -; CHECK-NEXT: [[TRUNC:%.*]] = shl <2 x i32> [[VAL_TR]], +; CHECK-NEXT: [[TRUNC:%.*]] = shl <2 x i32> [[VAL_TR]], ; CHECK-NEXT: ret <2 x i32> [[TRUNC]] ; - %shl = shl <2 x i64> %val, + %shl = shl <2 x i64> %val, %trunc = trunc <2 x i64> %shl to <2 x i32> ret <2 x i32> %trunc } @@ -917,7 +913,7 @@ define <4 x i8> @wide_shuf(<4 x i32> %x) { ret <4 x i8> %trunc } -; trunc (shuffle X, undef, SplatMask) --> shuffle (trunc X), undef, SplatMask +; trunc (shuffle X, poison, SplatMask) --> shuffle (trunc X), poison, SplatMask define <4 x i8> @wide_splat1(<4 x i32> %x) { ; CHECK-LABEL: @wide_splat1( @@ -931,7 +927,7 @@ define <4 x i8> @wide_splat1(<4 x i32> %x) { } ; Test weird types. -; trunc (shuffle X, undef, SplatMask) --> shuffle (trunc X), undef, SplatMask +; trunc (shuffle X, poison, SplatMask) --> shuffle (trunc X), poison, SplatMask define <3 x i31> @wide_splat2(<3 x i33> %x) { ; CHECK-LABEL: @wide_splat2( @@ -945,8 +941,8 @@ define <3 x i31> @wide_splat2(<3 x i33> %x) { } ; FIXME: -; trunc (shuffle X, undef, SplatMask) --> shuffle (trunc X), undef, SplatMask -; A mask with undef elements should still be considered a splat mask. +; trunc (shuffle X, poison, SplatMask) --> shuffle (trunc X), poison, SplatMask +; A mask with poison elements should still be considered a splat mask. define <3 x i31> @wide_splat3(<3 x i33> %x) { ; CHECK-LABEL: @wide_splat3( @@ -954,7 +950,7 @@ define <3 x i31> @wide_splat3(<3 x i33> %x) { ; CHECK-NEXT: [[TRUNC:%.*]] = trunc <3 x i33> [[SHUF]] to <3 x i31> ; CHECK-NEXT: ret <3 x i31> [[TRUNC]] ; - %shuf = shufflevector <3 x i33> %x, <3 x i33> poison, <3 x i32> + %shuf = shufflevector <3 x i33> %x, <3 x i33> poison, <3 x i32> %trunc = trunc <3 x i33> %shuf to <3 x i31> ret <3 x i31> %trunc } diff --git a/llvm/test/Transforms/InstCombine/trunc-shift-trunc.ll b/llvm/test/Transforms/InstCombine/trunc-shift-trunc.ll index 2c5f428cf98d..c50a3d06d24b 100644 --- a/llvm/test/Transforms/InstCombine/trunc-shift-trunc.ll +++ b/llvm/test/Transforms/InstCombine/trunc-shift-trunc.ll @@ -56,14 +56,14 @@ define <2 x i8> @trunc_lshr_trunc_nonuniform(<2 x i64> %a) { ret <2 x i8> %d } -define <2 x i8> @trunc_lshr_trunc_uniform_undef(<2 x i64> %a) { -; CHECK-LABEL: @trunc_lshr_trunc_uniform_undef( -; CHECK-NEXT: [[C1:%.*]] = lshr <2 x i64> [[A:%.*]], +define <2 x i8> @trunc_lshr_trunc_uniform_poison(<2 x i64> %a) { +; CHECK-LABEL: @trunc_lshr_trunc_uniform_poison( +; CHECK-NEXT: [[C1:%.*]] = lshr <2 x i64> [[A:%.*]], ; CHECK-NEXT: [[D:%.*]] = trunc <2 x i64> [[C1]] to <2 x i8> ; CHECK-NEXT: ret <2 x i8> [[D]] ; %b = trunc <2 x i64> %a to <2 x i32> - %c = lshr <2 x i32> %b, + %c = lshr <2 x i32> %b, %d = trunc <2 x i32> %c to <2 x i8> ret <2 x i8> %d } @@ -142,14 +142,14 @@ define <2 x i8> @trunc_ashr_trunc_nonuniform(<2 x i64> %a) { ret <2 x i8> %d } -define <2 x i8> @trunc_ashr_trunc_uniform_undef(<2 x i64> %a) { -; CHECK-LABEL: @trunc_ashr_trunc_uniform_undef( -; CHECK-NEXT: [[C1:%.*]] = ashr <2 x i64> [[A:%.*]], +define <2 x i8> @trunc_ashr_trunc_uniform_poison(<2 x i64> %a) { +; CHECK-LABEL: @trunc_ashr_trunc_uniform_poison( +; CHECK-NEXT: [[C1:%.*]] = ashr <2 x i64> [[A:%.*]], ; CHECK-NEXT: [[D:%.*]] = trunc <2 x i64> [[C1]] to <2 x i8> ; CHECK-NEXT: ret <2 x i8> [[D]] ; %b = trunc <2 x i64> %a to <2 x i32> - %c = ashr <2 x i32> %b, + %c = ashr <2 x i32> %b, %d = trunc <2 x i32> %c to <2 x i8> ret <2 x i8> %d } diff --git a/llvm/test/Transforms/InstCombine/trunc.ll b/llvm/test/Transforms/InstCombine/trunc.ll index c77d7269f2cf..e59b2bea6684 100644 --- a/llvm/test/Transforms/InstCombine/trunc.ll +++ b/llvm/test/Transforms/InstCombine/trunc.ll @@ -49,15 +49,15 @@ define <2 x i64> @test1_vec_nonuniform(<2 x i64> %a) { ret <2 x i64> %d } -define <2 x i64> @test1_vec_undef(<2 x i64> %a) { -; CHECK-LABEL: @test1_vec_undef( +define <2 x i64> @test1_vec_poison(<2 x i64> %a) { +; CHECK-LABEL: @test1_vec_poison( ; CHECK-NEXT: [[B:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i32> -; CHECK-NEXT: [[D:%.*]] = and <2 x i64> [[A]], +; CHECK-NEXT: [[D:%.*]] = and <2 x i64> [[A]], ; CHECK-NEXT: call void @use_vec(<2 x i32> [[B]]) ; CHECK-NEXT: ret <2 x i64> [[D]] ; %b = trunc <2 x i64> %a to <2 x i32> - %c = and <2 x i32> %b, + %c = and <2 x i32> %b, %d = zext <2 x i32> %c to <2 x i64> call void @use_vec(<2 x i32> %b) ret <2 x i64> %d @@ -111,17 +111,17 @@ define <2 x i64> @test2_vec_nonuniform(<2 x i64> %a) { ret <2 x i64> %d } -define <2 x i64> @test2_vec_undef(<2 x i64> %a) { -; CHECK-LABEL: @test2_vec_undef( +define <2 x i64> @test2_vec_poison(<2 x i64> %a) { +; CHECK-LABEL: @test2_vec_poison( ; CHECK-NEXT: [[B:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i32> -; CHECK-NEXT: [[D1:%.*]] = shl <2 x i64> [[A]], -; CHECK-NEXT: [[D:%.*]] = ashr exact <2 x i64> [[D1]], +; CHECK-NEXT: [[D1:%.*]] = shl <2 x i64> [[A]], +; CHECK-NEXT: [[D:%.*]] = ashr exact <2 x i64> [[D1]], ; CHECK-NEXT: call void @use_vec(<2 x i32> [[B]]) ; CHECK-NEXT: ret <2 x i64> [[D]] ; %b = trunc <2 x i64> %a to <2 x i32> - %c = shl <2 x i32> %b, - %q = ashr <2 x i32> %c, + %c = shl <2 x i32> %b, + %q = ashr <2 x i32> %c, %d = sext <2 x i32> %q to <2 x i64> call void @use_vec(<2 x i32> %b) ret <2 x i64> %d @@ -300,18 +300,17 @@ define <2 x i64> @test8_vec_nonuniform(<2 x i32> %A, <2 x i32> %B) { ret <2 x i64> %G } -define <2 x i64> @test8_vec_undef(<2 x i32> %A, <2 x i32> %B) { -; CHECK-LABEL: @test8_vec_undef( -; CHECK-NEXT: [[C:%.*]] = zext <2 x i32> [[A:%.*]] to <2 x i128> -; CHECK-NEXT: [[D:%.*]] = zext <2 x i32> [[B:%.*]] to <2 x i128> -; CHECK-NEXT: [[E:%.*]] = shl <2 x i128> [[D]], -; CHECK-NEXT: [[F:%.*]] = or <2 x i128> [[E]], [[C]] -; CHECK-NEXT: [[G:%.*]] = trunc <2 x i128> [[F]] to <2 x i64> +define <2 x i64> @test8_vec_poison(<2 x i32> %A, <2 x i32> %B) { +; CHECK-LABEL: @test8_vec_poison( +; CHECK-NEXT: [[C:%.*]] = zext <2 x i32> [[A:%.*]] to <2 x i64> +; CHECK-NEXT: [[D:%.*]] = zext <2 x i32> [[B:%.*]] to <2 x i64> +; CHECK-NEXT: [[E:%.*]] = shl nuw <2 x i64> [[D]], +; CHECK-NEXT: [[G:%.*]] = or disjoint <2 x i64> [[E]], [[C]] ; CHECK-NEXT: ret <2 x i64> [[G]] ; %C = zext <2 x i32> %A to <2 x i128> %D = zext <2 x i32> %B to <2 x i128> - %E = shl <2 x i128> %D, + %E = shl <2 x i128> %D, %F = or <2 x i128> %E, %C %G = trunc <2 x i128> %F to <2 x i64> ret <2 x i64> %G @@ -388,18 +387,17 @@ define <2 x i64> @test11_vec_nonuniform(<2 x i32> %A, <2 x i32> %B) { ret <2 x i64> %G } -define <2 x i64> @test11_vec_undef(<2 x i32> %A, <2 x i32> %B) { -; CHECK-LABEL: @test11_vec_undef( -; CHECK-NEXT: [[C:%.*]] = zext <2 x i32> [[A:%.*]] to <2 x i128> -; CHECK-NEXT: [[D:%.*]] = zext <2 x i32> [[B:%.*]] to <2 x i128> -; CHECK-NEXT: [[E:%.*]] = and <2 x i128> [[D]], -; CHECK-NEXT: [[F:%.*]] = shl <2 x i128> [[C]], [[E]] -; CHECK-NEXT: [[G:%.*]] = trunc <2 x i128> [[F]] to <2 x i64> +define <2 x i64> @test11_vec_poison(<2 x i32> %A, <2 x i32> %B) { +; CHECK-LABEL: @test11_vec_poison( +; CHECK-NEXT: [[C:%.*]] = zext <2 x i32> [[A:%.*]] to <2 x i64> +; CHECK-NEXT: [[TMP1:%.*]] = and <2 x i32> [[B:%.*]], +; CHECK-NEXT: [[E:%.*]] = zext nneg <2 x i32> [[TMP1]] to <2 x i64> +; CHECK-NEXT: [[G:%.*]] = shl nuw nsw <2 x i64> [[C]], [[E]] ; CHECK-NEXT: ret <2 x i64> [[G]] ; %C = zext <2 x i32> %A to <2 x i128> %D = zext <2 x i32> %B to <2 x i128> - %E = and <2 x i128> %D, + %E = and <2 x i128> %D, %F = shl <2 x i128> %C, %E %G = trunc <2 x i128> %F to <2 x i64> ret <2 x i64> %G @@ -453,18 +451,17 @@ define <2 x i64> @test12_vec_nonuniform(<2 x i32> %A, <2 x i32> %B) { ret <2 x i64> %G } -define <2 x i64> @test12_vec_undef(<2 x i32> %A, <2 x i32> %B) { -; CHECK-LABEL: @test12_vec_undef( -; CHECK-NEXT: [[C:%.*]] = zext <2 x i32> [[A:%.*]] to <2 x i128> -; CHECK-NEXT: [[D:%.*]] = zext <2 x i32> [[B:%.*]] to <2 x i128> -; CHECK-NEXT: [[E:%.*]] = and <2 x i128> [[D]], -; CHECK-NEXT: [[F:%.*]] = lshr <2 x i128> [[C]], [[E]] -; CHECK-NEXT: [[G:%.*]] = trunc nuw nsw <2 x i128> [[F]] to <2 x i64> +define <2 x i64> @test12_vec_poison(<2 x i32> %A, <2 x i32> %B) { +; CHECK-LABEL: @test12_vec_poison( +; CHECK-NEXT: [[C:%.*]] = zext <2 x i32> [[A:%.*]] to <2 x i64> +; CHECK-NEXT: [[TMP1:%.*]] = and <2 x i32> [[B:%.*]], +; CHECK-NEXT: [[E:%.*]] = zext nneg <2 x i32> [[TMP1]] to <2 x i64> +; CHECK-NEXT: [[G:%.*]] = lshr <2 x i64> [[C]], [[E]] ; CHECK-NEXT: ret <2 x i64> [[G]] ; %C = zext <2 x i32> %A to <2 x i128> %D = zext <2 x i32> %B to <2 x i128> - %E = and <2 x i128> %D, + %E = and <2 x i128> %D, %F = lshr <2 x i128> %C, %E %G = trunc <2 x i128> %F to <2 x i64> ret <2 x i64> %G @@ -518,18 +515,17 @@ define <2 x i64> @test13_vec_nonuniform(<2 x i32> %A, <2 x i32> %B) { ret <2 x i64> %G } -define <2 x i64> @test13_vec_undef(<2 x i32> %A, <2 x i32> %B) { -; CHECK-LABEL: @test13_vec_undef( -; CHECK-NEXT: [[C:%.*]] = sext <2 x i32> [[A:%.*]] to <2 x i128> -; CHECK-NEXT: [[D:%.*]] = zext <2 x i32> [[B:%.*]] to <2 x i128> -; CHECK-NEXT: [[E:%.*]] = and <2 x i128> [[D]], -; CHECK-NEXT: [[F:%.*]] = ashr <2 x i128> [[C]], [[E]] -; CHECK-NEXT: [[G:%.*]] = trunc nsw <2 x i128> [[F]] to <2 x i64> +define <2 x i64> @test13_vec_poison(<2 x i32> %A, <2 x i32> %B) { +; CHECK-LABEL: @test13_vec_poison( +; CHECK-NEXT: [[C:%.*]] = sext <2 x i32> [[A:%.*]] to <2 x i64> +; CHECK-NEXT: [[TMP1:%.*]] = and <2 x i32> [[B:%.*]], +; CHECK-NEXT: [[E:%.*]] = zext nneg <2 x i32> [[TMP1]] to <2 x i64> +; CHECK-NEXT: [[G:%.*]] = ashr <2 x i64> [[C]], [[E]] ; CHECK-NEXT: ret <2 x i64> [[G]] ; %C = sext <2 x i32> %A to <2 x i128> %D = zext <2 x i32> %B to <2 x i128> - %E = and <2 x i128> %D, + %E = and <2 x i128> %D, %F = ashr <2 x i128> %C, %E %G = trunc <2 x i128> %F to <2 x i64> ret <2 x i64> %G @@ -766,13 +762,13 @@ define <2 x i32> @trunc_shl_v2i32_v2i64_uniform(<2 x i64> %val) { ret <2 x i32> %trunc } -define <2 x i32> @trunc_shl_v2i32_v2i64_undef(<2 x i64> %val) { -; CHECK-LABEL: @trunc_shl_v2i32_v2i64_undef( +define <2 x i32> @trunc_shl_v2i32_v2i64_poison(<2 x i64> %val) { +; CHECK-LABEL: @trunc_shl_v2i32_v2i64_poison( ; CHECK-NEXT: [[VAL_TR:%.*]] = trunc <2 x i64> [[VAL:%.*]] to <2 x i32> -; CHECK-NEXT: [[TRUNC:%.*]] = shl <2 x i32> [[VAL_TR]], +; CHECK-NEXT: [[TRUNC:%.*]] = shl <2 x i32> [[VAL_TR]], ; CHECK-NEXT: ret <2 x i32> [[TRUNC]] ; - %shl = shl <2 x i64> %val, + %shl = shl <2 x i64> %val, %trunc = trunc <2 x i64> %shl to <2 x i32> ret <2 x i32> %trunc } @@ -917,7 +913,7 @@ define <4 x i8> @wide_shuf(<4 x i32> %x) { ret <4 x i8> %trunc } -; trunc (shuffle X, undef, SplatMask) --> shuffle (trunc X), undef, SplatMask +; trunc (shuffle X, poison, SplatMask) --> shuffle (trunc X), poison, SplatMask define <4 x i8> @wide_splat1(<4 x i32> %x) { ; CHECK-LABEL: @wide_splat1( @@ -925,13 +921,13 @@ define <4 x i8> @wide_splat1(<4 x i32> %x) { ; CHECK-NEXT: [[TRUNC:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i8> [[TRUNC]] ; - %shuf = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> + %shuf = shufflevector <4 x i32> %x, <4 x i32> poison, <4 x i32> %trunc = trunc <4 x i32> %shuf to <4 x i8> ret <4 x i8> %trunc } ; Test weird types. -; trunc (shuffle X, undef, SplatMask) --> shuffle (trunc X), undef, SplatMask +; trunc (shuffle X, poison, SplatMask) --> shuffle (trunc X), poison, SplatMask define <3 x i31> @wide_splat2(<3 x i33> %x) { ; CHECK-LABEL: @wide_splat2( @@ -939,14 +935,14 @@ define <3 x i31> @wide_splat2(<3 x i33> %x) { ; CHECK-NEXT: [[TRUNC:%.*]] = shufflevector <3 x i31> [[TMP1]], <3 x i31> poison, <3 x i32> ; CHECK-NEXT: ret <3 x i31> [[TRUNC]] ; - %shuf = shufflevector <3 x i33> %x, <3 x i33> undef, <3 x i32> + %shuf = shufflevector <3 x i33> %x, <3 x i33> poison, <3 x i32> %trunc = trunc <3 x i33> %shuf to <3 x i31> ret <3 x i31> %trunc } ; FIXME: -; trunc (shuffle X, undef, SplatMask) --> shuffle (trunc X), undef, SplatMask -; A mask with undef elements should still be considered a splat mask. +; trunc (shuffle X, poison, SplatMask) --> shuffle (trunc X), poison, SplatMask +; A mask with poison elements should still be considered a splat mask. define <3 x i31> @wide_splat3(<3 x i33> %x) { ; CHECK-LABEL: @wide_splat3( @@ -954,7 +950,7 @@ define <3 x i31> @wide_splat3(<3 x i33> %x) { ; CHECK-NEXT: [[TRUNC:%.*]] = trunc <3 x i33> [[SHUF]] to <3 x i31> ; CHECK-NEXT: ret <3 x i31> [[TRUNC]] ; - %shuf = shufflevector <3 x i33> %x, <3 x i33> undef, <3 x i32> + %shuf = shufflevector <3 x i33> %x, <3 x i33> poison, <3 x i32> %trunc = trunc <3 x i33> %shuf to <3 x i31> ret <3 x i31> %trunc } diff --git a/llvm/test/Transforms/InstCombine/unsigned-mul-lack-of-overflow-check-via-udiv-of-allones.ll b/llvm/test/Transforms/InstCombine/unsigned-mul-lack-of-overflow-check-via-udiv-of-allones.ll index 1ffcfb4424e3..241d9cbcde33 100644 --- a/llvm/test/Transforms/InstCombine/unsigned-mul-lack-of-overflow-check-via-udiv-of-allones.ll +++ b/llvm/test/Transforms/InstCombine/unsigned-mul-lack-of-overflow-check-via-udiv-of-allones.ll @@ -30,14 +30,14 @@ define <2 x i1> @t1_vec(<2 x i8> %x, <2 x i8> %y) { ret <2 x i1> %r } -define <3 x i1> @t2_vec_undef(<3 x i8> %x, <3 x i8> %y) { -; CHECK-LABEL: @t2_vec_undef( +define <3 x i1> @t2_vec_poison(<3 x i8> %x, <3 x i8> %y) { +; CHECK-LABEL: @t2_vec_poison( ; CHECK-NEXT: [[MUL:%.*]] = call { <3 x i8>, <3 x i1> } @llvm.umul.with.overflow.v3i8(<3 x i8> [[X:%.*]], <3 x i8> [[Y:%.*]]) ; CHECK-NEXT: [[MUL_OV:%.*]] = extractvalue { <3 x i8>, <3 x i1> } [[MUL]], 1 ; CHECK-NEXT: [[MUL_NOT_OV:%.*]] = xor <3 x i1> [[MUL_OV]], ; CHECK-NEXT: ret <3 x i1> [[MUL_NOT_OV]] ; - %t0 = udiv <3 x i8> , %x + %t0 = udiv <3 x i8> , %x %r = icmp uge <3 x i8> %t0, %y ret <3 x i1> %r } diff --git a/llvm/test/Transforms/InstCombine/unsigned-mul-overflow-check-via-udiv-of-allones.ll b/llvm/test/Transforms/InstCombine/unsigned-mul-overflow-check-via-udiv-of-allones.ll index 710a09f6e16a..7eb08bdd6016 100644 --- a/llvm/test/Transforms/InstCombine/unsigned-mul-overflow-check-via-udiv-of-allones.ll +++ b/llvm/test/Transforms/InstCombine/unsigned-mul-overflow-check-via-udiv-of-allones.ll @@ -28,13 +28,13 @@ define <2 x i1> @t1_vec(<2 x i8> %x, <2 x i8> %y) { ret <2 x i1> %r } -define <3 x i1> @t2_vec_undef(<3 x i8> %x, <3 x i8> %y) { -; CHECK-LABEL: @t2_vec_undef( +define <3 x i1> @t2_vec_poison(<3 x i8> %x, <3 x i8> %y) { +; CHECK-LABEL: @t2_vec_poison( ; CHECK-NEXT: [[MUL:%.*]] = call { <3 x i8>, <3 x i1> } @llvm.umul.with.overflow.v3i8(<3 x i8> [[X:%.*]], <3 x i8> [[Y:%.*]]) ; CHECK-NEXT: [[MUL_OV:%.*]] = extractvalue { <3 x i8>, <3 x i1> } [[MUL]], 1 ; CHECK-NEXT: ret <3 x i1> [[MUL_OV]] ; - %t0 = udiv <3 x i8> , %x + %t0 = udiv <3 x i8> , %x %r = icmp ult <3 x i8> %t0, %y ret <3 x i1> %r } diff --git a/llvm/test/Transforms/InstCombine/variable-signext-of-variable-high-bit-extraction.ll b/llvm/test/Transforms/InstCombine/variable-signext-of-variable-high-bit-extraction.ll index adacf3ce99b2..262942aa1219 100644 --- a/llvm/test/Transforms/InstCombine/variable-signext-of-variable-high-bit-extraction.ll +++ b/llvm/test/Transforms/InstCombine/variable-signext-of-variable-high-bit-extraction.ll @@ -203,20 +203,20 @@ define <2 x i32> @t4_vec(<2 x i64> %data, <2 x i32> %nbits) { ret <2 x i32> %signextended } -define <3 x i32> @t5_vec_undef(<3 x i64> %data, <3 x i32> %nbits) { -; CHECK-LABEL: @t5_vec_undef( -; CHECK-NEXT: [[SKIP_HIGH:%.*]] = sub <3 x i32> , [[NBITS:%.*]] +define <3 x i32> @t5_vec_poison(<3 x i64> %data, <3 x i32> %nbits) { +; CHECK-LABEL: @t5_vec_poison( +; CHECK-NEXT: [[SKIP_HIGH:%.*]] = sub <3 x i32> , [[NBITS:%.*]] ; CHECK-NEXT: [[SKIP_HIGH_WIDE:%.*]] = zext nneg <3 x i32> [[SKIP_HIGH]] to <3 x i64> ; CHECK-NEXT: [[TMP1:%.*]] = ashr <3 x i64> [[DATA:%.*]], [[SKIP_HIGH_WIDE]] ; CHECK-NEXT: [[SIGNEXTENDED:%.*]] = trunc <3 x i64> [[TMP1]] to <3 x i32> ; CHECK-NEXT: ret <3 x i32> [[SIGNEXTENDED]] ; - %skip_high = sub <3 x i32> , %nbits + %skip_high = sub <3 x i32> , %nbits %skip_high_wide = zext <3 x i32> %skip_high to <3 x i64> %extracted = lshr <3 x i64> %data, %skip_high_wide %extracted_narrow = trunc <3 x i64> %extracted to <3 x i32> - %num_high_bits_to_smear_narrow0 = sub <3 x i32> , %nbits - %num_high_bits_to_smear_narrow1 = sub <3 x i32> , %nbits + %num_high_bits_to_smear_narrow0 = sub <3 x i32> , %nbits + %num_high_bits_to_smear_narrow1 = sub <3 x i32> , %nbits %signbit_positioned = shl <3 x i32> %extracted_narrow, %num_high_bits_to_smear_narrow0 %signextended = ashr <3 x i32> %signbit_positioned, %num_high_bits_to_smear_narrow1 ret <3 x i32> %signextended diff --git a/llvm/test/Transforms/InstCombine/vec_sext.ll b/llvm/test/Transforms/InstCombine/vec_sext.ll index a880d5e56272..9f5f957f4944 100644 --- a/llvm/test/Transforms/InstCombine/vec_sext.ll +++ b/llvm/test/Transforms/InstCombine/vec_sext.ll @@ -42,24 +42,24 @@ define <4 x i32> @vec_select_alternate_sign_bit_test(<4 x i32> %a, <4 x i32> %b) ret <4 x i32> %cond } -define <2 x i32> @is_negative_undef_elt(<2 x i32> %a) { -; CHECK-LABEL: @is_negative_undef_elt( +define <2 x i32> @is_negative_poison_elt(<2 x i32> %a) { +; CHECK-LABEL: @is_negative_poison_elt( ; CHECK-NEXT: [[A_LOBIT:%.*]] = ashr <2 x i32> [[A:%.*]], ; CHECK-NEXT: ret <2 x i32> [[A_LOBIT]] ; - %cmp = icmp slt <2 x i32> %a, + %cmp = icmp slt <2 x i32> %a, %sext = sext <2 x i1> %cmp to <2 x i32> ret <2 x i32> %sext } -define <2 x i32> @is_positive_undef_elt(<2 x i32> %a) { -; CHECK-LABEL: @is_positive_undef_elt( -; CHECK-NEXT: [[CMP:%.*]] = icmp sgt <2 x i32> [[A:%.*]], +define <2 x i32> @is_positive_poison_elt(<2 x i32> %a) { +; CHECK-LABEL: @is_positive_poison_elt( +; CHECK-NEXT: [[CMP:%.*]] = icmp sgt <2 x i32> [[A:%.*]], ; CHECK-NEXT: [[SEXT:%.*]] = sext <2 x i1> [[CMP]] to <2 x i32> ; CHECK-NEXT: ret <2 x i32> [[SEXT]] ; - %cmp = icmp sgt <2 x i32> %a, + %cmp = icmp sgt <2 x i32> %a, %sext = sext <2 x i1> %cmp to <2 x i32> ret <2 x i32> %sext } diff --git a/llvm/test/Transforms/InstCombine/vector-casts-inseltpoison.ll b/llvm/test/Transforms/InstCombine/vector-casts-inseltpoison.ll index cf1b72fbcf3e..a87364600ba3 100644 --- a/llvm/test/Transforms/InstCombine/vector-casts-inseltpoison.ll +++ b/llvm/test/Transforms/InstCombine/vector-casts-inseltpoison.ll @@ -26,26 +26,26 @@ define <2 x i1> @and_cmp_is_trunc(<2 x i64> %a) { ; This is trunc. -define <2 x i1> @and_cmp_is_trunc_even_with_undef_elt(<2 x i64> %a) { -; CHECK-LABEL: @and_cmp_is_trunc_even_with_undef_elt( +define <2 x i1> @and_cmp_is_trunc_even_with_poison_elt(<2 x i64> %a) { +; CHECK-LABEL: @and_cmp_is_trunc_even_with_poison_elt( ; CHECK-NEXT: [[R:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i1> ; CHECK-NEXT: ret <2 x i1> [[R]] ; - %t = and <2 x i64> %a, + %t = and <2 x i64> %a, %r = icmp ne <2 x i64> %t, zeroinitializer ret <2 x i1> %r } -; TODO: This could be just 1 instruction (trunc), but our undef matching is incomplete. +; TODO: This could be just 1 instruction (trunc), but our poison matching is incomplete. -define <2 x i1> @and_cmp_is_trunc_even_with_undef_elts(<2 x i64> %a) { -; CHECK-LABEL: @and_cmp_is_trunc_even_with_undef_elts( -; CHECK-NEXT: [[T:%.*]] = and <2 x i64> [[A:%.*]], -; CHECK-NEXT: [[R:%.*]] = icmp ne <2 x i64> [[T]], +define <2 x i1> @and_cmp_is_trunc_even_with_poison_elts(<2 x i64> %a) { +; CHECK-LABEL: @and_cmp_is_trunc_even_with_poison_elts( +; CHECK-NEXT: [[T:%.*]] = and <2 x i64> [[A:%.*]], +; CHECK-NEXT: [[R:%.*]] = icmp ne <2 x i64> [[T]], ; CHECK-NEXT: ret <2 x i1> [[R]] ; - %t = and <2 x i64> %a, - %r = icmp ne <2 x i64> %t, + %t = and <2 x i64> %a, + %r = icmp ne <2 x i64> %t, ret <2 x i1> %r } diff --git a/llvm/test/Transforms/InstCombine/vector-casts.ll b/llvm/test/Transforms/InstCombine/vector-casts.ll index 281fc5f6011e..fd2a4ffdfb70 100644 --- a/llvm/test/Transforms/InstCombine/vector-casts.ll +++ b/llvm/test/Transforms/InstCombine/vector-casts.ll @@ -26,26 +26,26 @@ define <2 x i1> @and_cmp_is_trunc(<2 x i64> %a) { ; This is trunc. -define <2 x i1> @and_cmp_is_trunc_even_with_undef_elt(<2 x i64> %a) { -; CHECK-LABEL: @and_cmp_is_trunc_even_with_undef_elt( +define <2 x i1> @and_cmp_is_trunc_even_with_poison_elt(<2 x i64> %a) { +; CHECK-LABEL: @and_cmp_is_trunc_even_with_poison_elt( ; CHECK-NEXT: [[R:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i1> ; CHECK-NEXT: ret <2 x i1> [[R]] ; - %t = and <2 x i64> %a, + %t = and <2 x i64> %a, %r = icmp ne <2 x i64> %t, zeroinitializer ret <2 x i1> %r } -; TODO: This could be just 1 instruction (trunc), but our undef matching is incomplete. +; TODO: This could be just 1 instruction (trunc), but our poison matching is incomplete. -define <2 x i1> @and_cmp_is_trunc_even_with_undef_elts(<2 x i64> %a) { -; CHECK-LABEL: @and_cmp_is_trunc_even_with_undef_elts( -; CHECK-NEXT: [[T:%.*]] = and <2 x i64> [[A:%.*]], -; CHECK-NEXT: [[R:%.*]] = icmp ne <2 x i64> [[T]], +define <2 x i1> @and_cmp_is_trunc_even_with_poison_elts(<2 x i64> %a) { +; CHECK-LABEL: @and_cmp_is_trunc_even_with_poison_elts( +; CHECK-NEXT: [[T:%.*]] = and <2 x i64> [[A:%.*]], +; CHECK-NEXT: [[R:%.*]] = icmp ne <2 x i64> [[T]], ; CHECK-NEXT: ret <2 x i1> [[R]] ; - %t = and <2 x i64> %a, - %r = icmp ne <2 x i64> %t, + %t = and <2 x i64> %a, + %r = icmp ne <2 x i64> %t, ret <2 x i1> %r } diff --git a/llvm/test/Transforms/InstCombine/vector-urem.ll b/llvm/test/Transforms/InstCombine/vector-urem.ll index d5c77470a20f..627789a03ef6 100644 --- a/llvm/test/Transforms/InstCombine/vector-urem.ll +++ b/llvm/test/Transforms/InstCombine/vector-urem.ll @@ -19,11 +19,11 @@ define <4 x i32> @test_v4i32_const_pow2(<4 x i32> %a0) { ret <4 x i32> %1 } -define <4 x i32> @test_v4i32_const_pow2_undef(<4 x i32> %a0) { -; CHECK-LABEL: @test_v4i32_const_pow2_undef( +define <4 x i32> @test_v4i32_const_pow2_poison(<4 x i32> %a0) { +; CHECK-LABEL: @test_v4i32_const_pow2_poison( ; CHECK-NEXT: ret <4 x i32> poison ; - %1 = urem <4 x i32> %a0, + %1 = urem <4 x i32> %a0, ret <4 x i32> %1 } @@ -37,13 +37,13 @@ define <4 x i32> @test_v4i32_one(<4 x i32> %a0) { ret <4 x i32> %1 } -define <4 x i32> @test_v4i32_one_undef(<4 x i32> %a0) { -; CHECK-LABEL: @test_v4i32_one_undef( +define <4 x i32> @test_v4i32_one_poison(<4 x i32> %a0) { +; CHECK-LABEL: @test_v4i32_one_poison( ; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <4 x i32> [[A0:%.*]], ; CHECK-NEXT: [[TMP2:%.*]] = zext <4 x i1> [[TMP1]] to <4 x i32> ; CHECK-NEXT: ret <4 x i32> [[TMP2]] ; - %1 = urem <4 x i32> , %a0 + %1 = urem <4 x i32> , %a0 ret <4 x i32> %1 } @@ -71,10 +71,10 @@ define <4 x i32> @test_v4i32_negconst(<4 x i32> %a0) { ret <4 x i32> %1 } -define <4 x i32> @test_v4i32_negconst_undef(<4 x i32> %a0) { -; CHECK-LABEL: @test_v4i32_negconst_undef( +define <4 x i32> @test_v4i32_negconst_poison(<4 x i32> %a0) { +; CHECK-LABEL: @test_v4i32_negconst_poison( ; CHECK-NEXT: ret <4 x i32> poison ; - %1 = urem <4 x i32> %a0, + %1 = urem <4 x i32> %a0, ret <4 x i32> %1 } diff --git a/llvm/test/Transforms/InstCombine/vector-xor.ll b/llvm/test/Transforms/InstCombine/vector-xor.ll index 171dd6e35b4e..ee593b5d15e8 100644 --- a/llvm/test/Transforms/InstCombine/vector-xor.ll +++ b/llvm/test/Transforms/InstCombine/vector-xor.ll @@ -53,14 +53,14 @@ define <4 x i32> @test_v4i32_xor_bswap_const(<4 x i32> %a0) { ret <4 x i32> %2 } -define <4 x i32> @test_v4i32_xor_bswap_const_undef(<4 x i32> %a0) { -; CHECK-LABEL: @test_v4i32_xor_bswap_const_undef( +define <4 x i32> @test_v4i32_xor_bswap_const_poison(<4 x i32> %a0) { +; CHECK-LABEL: @test_v4i32_xor_bswap_const_poison( ; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> [[A0:%.*]]) -; CHECK-NEXT: [[TMP2:%.*]] = xor <4 x i32> [[TMP1]], +; CHECK-NEXT: [[TMP2:%.*]] = xor <4 x i32> [[TMP1]], ; CHECK-NEXT: ret <4 x i32> [[TMP2]] ; %1 = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> %a0) - %2 = xor <4 x i32> %1, + %2 = xor <4 x i32> %1, ret <4 x i32> %2 } @@ -105,14 +105,14 @@ define <4 x i32> @test_v4i32_not_ashr_not(<4 x i32> %x, <4 x i32> %y) { ret <4 x i32> %3 } -define <4 x i32> @test_v4i32_not_ashr_not_undef(<4 x i32> %x, <4 x i32> %y) { -; CHECK-LABEL: @test_v4i32_not_ashr_not_undef( +define <4 x i32> @test_v4i32_not_ashr_not_poison(<4 x i32> %x, <4 x i32> %y) { +; CHECK-LABEL: @test_v4i32_not_ashr_not_poison( ; CHECK-NEXT: [[DOTNOT:%.*]] = ashr <4 x i32> [[X:%.*]], [[Y:%.*]] ; CHECK-NEXT: ret <4 x i32> [[DOTNOT]] ; - %1 = xor <4 x i32> , %x + %1 = xor <4 x i32> , %x %2 = ashr <4 x i32> %1, %y - %3 = xor <4 x i32> , %2 + %3 = xor <4 x i32> , %2 ret <4 x i32> %3 } @@ -138,13 +138,13 @@ define <4 x i32> @test_v4i32_not_ashr_negative_const(<4 x i32> %a0) { ret <4 x i32> %2 } -define <4 x i32> @test_v4i32_not_ashr_negative_const_undef(<4 x i32> %a0) { -; CHECK-LABEL: @test_v4i32_not_ashr_negative_const_undef( +define <4 x i32> @test_v4i32_not_ashr_negative_const_poison(<4 x i32> %a0) { +; CHECK-LABEL: @test_v4i32_not_ashr_negative_const_poison( ; CHECK-NEXT: [[TMP1:%.*]] = lshr <4 x i32> , [[A0:%.*]] ; CHECK-NEXT: ret <4 x i32> [[TMP1]] ; - %1 = ashr <4 x i32> , %a0 - %2 = xor <4 x i32> , %1 + %1 = ashr <4 x i32> , %a0 + %2 = xor <4 x i32> , %1 ret <4 x i32> %2 } @@ -170,13 +170,13 @@ define <4 x i32> @test_v4i32_not_lshr_nonnegative_const(<4 x i32> %a0) { ret <4 x i32> %2 } -define <4 x i32> @test_v4i32_not_lshr_nonnegative_const_undef(<4 x i32> %a0) { -; CHECK-LABEL: @test_v4i32_not_lshr_nonnegative_const_undef( +define <4 x i32> @test_v4i32_not_lshr_nonnegative_const_poison(<4 x i32> %a0) { +; CHECK-LABEL: @test_v4i32_not_lshr_nonnegative_const_poison( ; CHECK-NEXT: [[TMP1:%.*]] = ashr <4 x i32> , [[A0:%.*]] ; CHECK-NEXT: ret <4 x i32> [[TMP1]] ; - %1 = lshr <4 x i32> , %a0 - %2 = xor <4 x i32> , %1 + %1 = lshr <4 x i32> , %a0 + %2 = xor <4 x i32> , %1 ret <4 x i32> %2 } @@ -202,13 +202,13 @@ define <4 x i32> @test_v4i32_not_sub_const(<4 x i32> %a0) { ret <4 x i32> %2 } -define <4 x i32> @test_v4i32_not_sub_const_undef(<4 x i32> %a0) { -; CHECK-LABEL: @test_v4i32_not_sub_const_undef( -; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[A0:%.*]], +define <4 x i32> @test_v4i32_not_sub_const_poison(<4 x i32> %a0) { +; CHECK-LABEL: @test_v4i32_not_sub_const_poison( +; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[A0:%.*]], ; CHECK-NEXT: ret <4 x i32> [[TMP1]] ; - %1 = sub <4 x i32> , %a0 - %2 = xor <4 x i32> , %1 + %1 = sub <4 x i32> , %a0 + %2 = xor <4 x i32> , %1 ret <4 x i32> %2 } @@ -235,14 +235,14 @@ define <4 x i32> @test_v4i32_xor_signmask_sub_const(<4 x i32> %a0) { ret <4 x i32> %2 } -define <4 x i32> @test_v4i32_xor_signmask_sub_const_undef(<4 x i32> %a0) { -; CHECK-LABEL: @test_v4i32_xor_signmask_sub_const_undef( -; CHECK-NEXT: [[TMP1:%.*]] = sub <4 x i32> , [[A0:%.*]] -; CHECK-NEXT: [[TMP2:%.*]] = xor <4 x i32> [[TMP1]], +define <4 x i32> @test_v4i32_xor_signmask_sub_const_poison(<4 x i32> %a0) { +; CHECK-LABEL: @test_v4i32_xor_signmask_sub_const_poison( +; CHECK-NEXT: [[TMP1:%.*]] = sub <4 x i32> , [[A0:%.*]] +; CHECK-NEXT: [[TMP2:%.*]] = xor <4 x i32> [[TMP1]], ; CHECK-NEXT: ret <4 x i32> [[TMP2]] ; - %1 = sub <4 x i32> , %a0 - %2 = xor <4 x i32> , %1 + %1 = sub <4 x i32> , %a0 + %2 = xor <4 x i32> , %1 ret <4 x i32> %2 } @@ -269,13 +269,13 @@ define <4 x i32> @test_v4i32_xor_signmask_add_const(<4 x i32> %a0) { ret <4 x i32> %2 } -define <4 x i32> @test_v4i32_xor_signmask_add_const_undef(<4 x i32> %a0) { -; CHECK-LABEL: @test_v4i32_xor_signmask_add_const_undef( -; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[A0:%.*]], -; CHECK-NEXT: [[TMP2:%.*]] = xor <4 x i32> [[TMP1]], +define <4 x i32> @test_v4i32_xor_signmask_add_const_poison(<4 x i32> %a0) { +; CHECK-LABEL: @test_v4i32_xor_signmask_add_const_poison( +; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[A0:%.*]], +; CHECK-NEXT: [[TMP2:%.*]] = xor <4 x i32> [[TMP1]], ; CHECK-NEXT: ret <4 x i32> [[TMP2]] ; - %1 = add <4 x i32> , %a0 - %2 = xor <4 x i32> , %1 + %1 = add <4 x i32> , %a0 + %2 = xor <4 x i32> , %1 ret <4 x i32> %2 } diff --git a/llvm/test/Transforms/InstCombine/zext-bool-add-sub.ll b/llvm/test/Transforms/InstCombine/zext-bool-add-sub.ll index 7fed952a7ff7..12739b5686a0 100644 --- a/llvm/test/Transforms/InstCombine/zext-bool-add-sub.ll +++ b/llvm/test/Transforms/InstCombine/zext-bool-add-sub.ll @@ -126,13 +126,13 @@ define <2 x i64> @zext_negate_vec(<2 x i1> %A) { ret <2 x i64> %sub } -define <2 x i64> @zext_negate_vec_undef_elt(<2 x i1> %A) { -; CHECK-LABEL: @zext_negate_vec_undef_elt( +define <2 x i64> @zext_negate_vec_poison_elt(<2 x i1> %A) { +; CHECK-LABEL: @zext_negate_vec_poison_elt( ; CHECK-NEXT: [[EXT_NEG:%.*]] = sext <2 x i1> [[A:%.*]] to <2 x i64> ; CHECK-NEXT: ret <2 x i64> [[EXT_NEG]] ; %ext = zext <2 x i1> %A to <2 x i64> - %sub = sub <2 x i64> , %ext + %sub = sub <2 x i64> , %ext ret <2 x i64> %sub } @@ -169,13 +169,13 @@ define <2 x i64> @zext_sub_const_vec(<2 x i1> %A) { ret <2 x i64> %sub } -define <2 x i64> @zext_sub_const_vec_undef_elt(<2 x i1> %A) { -; CHECK-LABEL: @zext_sub_const_vec_undef_elt( -; CHECK-NEXT: [[SUB:%.*]] = select <2 x i1> [[A:%.*]], <2 x i64> , <2 x i64> +define <2 x i64> @zext_sub_const_vec_poison_elt(<2 x i1> %A) { +; CHECK-LABEL: @zext_sub_const_vec_poison_elt( +; CHECK-NEXT: [[SUB:%.*]] = select <2 x i1> [[A:%.*]], <2 x i64> , <2 x i64> ; CHECK-NEXT: ret <2 x i64> [[SUB]] ; %ext = zext <2 x i1> %A to <2 x i64> - %sub = sub <2 x i64> , %ext + %sub = sub <2 x i64> , %ext ret <2 x i64> %sub } @@ -212,13 +212,13 @@ define <2 x i64> @sext_negate_vec(<2 x i1> %A) { ret <2 x i64> %sub } -define <2 x i64> @sext_negate_vec_undef_elt(<2 x i1> %A) { -; CHECK-LABEL: @sext_negate_vec_undef_elt( +define <2 x i64> @sext_negate_vec_poison_elt(<2 x i1> %A) { +; CHECK-LABEL: @sext_negate_vec_poison_elt( ; CHECK-NEXT: [[EXT_NEG:%.*]] = zext <2 x i1> [[A:%.*]] to <2 x i64> ; CHECK-NEXT: ret <2 x i64> [[EXT_NEG]] ; %ext = sext <2 x i1> %A to <2 x i64> - %sub = sub <2 x i64> , %ext + %sub = sub <2 x i64> , %ext ret <2 x i64> %sub } @@ -255,13 +255,13 @@ define <2 x i64> @sext_sub_const_vec(<2 x i1> %A) { ret <2 x i64> %sub } -define <2 x i64> @sext_sub_const_vec_undef_elt(<2 x i1> %A) { -; CHECK-LABEL: @sext_sub_const_vec_undef_elt( -; CHECK-NEXT: [[SUB:%.*]] = select <2 x i1> [[A:%.*]], <2 x i64> , <2 x i64> +define <2 x i64> @sext_sub_const_vec_poison_elt(<2 x i1> %A) { +; CHECK-LABEL: @sext_sub_const_vec_poison_elt( +; CHECK-NEXT: [[SUB:%.*]] = select <2 x i1> [[A:%.*]], <2 x i64> , <2 x i64> ; CHECK-NEXT: ret <2 x i64> [[SUB]] ; %ext = sext <2 x i1> %A to <2 x i64> - %sub = sub <2 x i64> , %ext + %sub = sub <2 x i64> , %ext ret <2 x i64> %sub } diff --git a/llvm/test/Transforms/InstSimplify/AndOrXor.ll b/llvm/test/Transforms/InstSimplify/AndOrXor.ll index 494b6bcd2b66..2e3a60522420 100644 --- a/llvm/test/Transforms/InstSimplify/AndOrXor.ll +++ b/llvm/test/Transforms/InstSimplify/AndOrXor.ll @@ -12,11 +12,11 @@ define i8 @and0(i8 %x) { ret i8 %r } -define <2 x i8> @and0_vec_undef_elt(<2 x i8> %x) { -; CHECK-LABEL: @and0_vec_undef_elt( +define <2 x i8> @and0_vec_poison_elt(<2 x i8> %x) { +; CHECK-LABEL: @and0_vec_poison_elt( ; CHECK-NEXT: ret <2 x i8> zeroinitializer ; - %r = and <2 x i8> %x, + %r = and <2 x i8> %x, ret <2 x i8> %r } @@ -31,14 +31,14 @@ define <2 x i32> @add_nsw_signbit(<2 x i32> %x) { ret <2 x i32> %z } -; Undef elements in either constant vector are ok. +; Poison elements in either constant vector are ok. -define <2 x i32> @add_nsw_signbit_undef(<2 x i32> %x) { -; CHECK-LABEL: @add_nsw_signbit_undef( +define <2 x i32> @add_nsw_signbit_poison(<2 x i32> %x) { +; CHECK-LABEL: @add_nsw_signbit_poison( ; CHECK-NEXT: ret <2 x i32> [[X:%.*]] ; - %y = xor <2 x i32> %x, - %z = add nsw <2 x i32> %y, + %y = xor <2 x i32> %x, + %z = add nsw <2 x i32> %y, ret <2 x i32> %z } @@ -53,14 +53,14 @@ define <2 x i5> @add_nuw_signbit(<2 x i5> %x) { ret <2 x i5> %z } -; Undef elements in either constant vector are ok. +; Poison elements in either constant vector are ok. -define <2 x i5> @add_nuw_signbit_undef(<2 x i5> %x) { -; CHECK-LABEL: @add_nuw_signbit_undef( +define <2 x i5> @add_nuw_signbit_poison(<2 x i5> %x) { +; CHECK-LABEL: @add_nuw_signbit_poison( ; CHECK-NEXT: ret <2 x i5> [[X:%.*]] ; - %y = xor <2 x i5> %x, - %z = add nuw <2 x i5> %y, + %y = xor <2 x i5> %x, + %z = add nuw <2 x i5> %y, ret <2 x i5> %z } @@ -584,7 +584,7 @@ define <2 x i32> @or_xor_andn_commute2(<2 x i32> %a, <2 x i32> %b) { ; CHECK-NEXT: ret <2 x i32> [[XOR]] ; %xor = xor <2 x i32> %a, %b - %neg = xor <2 x i32> %b, + %neg = xor <2 x i32> %b, %and = and <2 x i32> %a, %neg %or = or <2 x i32> %xor, %and ret <2 x i32> %or @@ -708,15 +708,13 @@ define <2 x i32> @or_xorn_and_commute2_undef(<2 x i32> %a, <2 x i32> %b) { ret <2 x i32> %or } -; TODO: Unlike the above test, this is safe to fold. +; Unlike the above test, this is safe to fold. define <2 x i32> @or_xorn_and_commute2_poison(<2 x i32> %a, <2 x i32> %b) { ; CHECK-LABEL: @or_xorn_and_commute2_poison( ; CHECK-NEXT: [[NEGA:%.*]] = xor <2 x i32> [[A:%.*]], -; CHECK-NEXT: [[AND:%.*]] = and <2 x i32> [[B:%.*]], [[A]] -; CHECK-NEXT: [[XOR:%.*]] = xor <2 x i32> [[B]], [[NEGA]] -; CHECK-NEXT: [[OR:%.*]] = or <2 x i32> [[XOR]], [[AND]] -; CHECK-NEXT: ret <2 x i32> [[OR]] +; CHECK-NEXT: [[XOR:%.*]] = xor <2 x i32> [[B:%.*]], [[NEGA]] +; CHECK-NEXT: ret <2 x i32> [[XOR]] ; %nega = xor <2 x i32> %a, %and = and <2 x i32> %b, %a diff --git a/llvm/test/Transforms/InstSimplify/call.ll b/llvm/test/Transforms/InstSimplify/call.ll index 52c207a27604..c6f6b65f89dc 100644 --- a/llvm/test/Transforms/InstSimplify/call.ll +++ b/llvm/test/Transforms/InstSimplify/call.ll @@ -976,7 +976,7 @@ define <2 x i8> @fshr_zero_vec(<2 x i8> %shamt) { ; CHECK-LABEL: @fshr_zero_vec( ; CHECK-NEXT: ret <2 x i8> zeroinitializer ; - %r = call <2 x i8> @llvm.fshr.v2i8(<2 x i8> zeroinitializer, <2 x i8> , <2 x i8> %shamt) + %r = call <2 x i8> @llvm.fshr.v2i8(<2 x i8> zeroinitializer, <2 x i8> , <2 x i8> %shamt) ret <2 x i8> %r } @@ -984,7 +984,7 @@ define <2 x i7> @fshl_ones_vec(<2 x i7> %shamt) { ; CHECK-LABEL: @fshl_ones_vec( ; CHECK-NEXT: ret <2 x i7> ; - %r = call <2 x i7> @llvm.fshl.v2i7(<2 x i7> , <2 x i7> , <2 x i7> %shamt) + %r = call <2 x i7> @llvm.fshl.v2i7(<2 x i7> , <2 x i7> , <2 x i7> %shamt) ret <2 x i7> %r } @@ -1466,7 +1466,7 @@ define <3 x i33> @cttz_shl1_vec(<3 x i33> %x) { ; CHECK-LABEL: @cttz_shl1_vec( ; CHECK-NEXT: ret <3 x i33> [[X:%.*]] ; - %s = shl <3 x i33> , %x + %s = shl <3 x i33> , %x %r = call <3 x i33> @llvm.cttz.v3i33(<3 x i33> %s, i1 false) ret <3 x i33> %r } @@ -1509,7 +1509,7 @@ define <3 x i33> @ctlz_lshr_sign_bit_vec(<3 x i33> %x) { ; CHECK-LABEL: @ctlz_lshr_sign_bit_vec( ; CHECK-NEXT: ret <3 x i33> [[X:%.*]] ; - %s = lshr <3 x i33> , %x + %s = lshr <3 x i33> , %x %r = call <3 x i33> @llvm.ctlz.v3i33(<3 x i33> %s, i1 false) ret <3 x i33> %r } @@ -1549,7 +1549,7 @@ define <3 x i33> @ctlz_ashr_sign_bit_vec(<3 x i33> %x) { ; CHECK-LABEL: @ctlz_ashr_sign_bit_vec( ; CHECK-NEXT: ret <3 x i33> zeroinitializer ; - %s = ashr <3 x i33> , %x + %s = ashr <3 x i33> , %x %r = call <3 x i33> @llvm.ctlz.v3i33(<3 x i33> %s, i1 true) ret <3 x i33> %r } diff --git a/llvm/test/Transforms/InstSimplify/compare.ll b/llvm/test/Transforms/InstSimplify/compare.ll index 1e90f0edbd80..724912d90bd8 100644 --- a/llvm/test/Transforms/InstSimplify/compare.ll +++ b/llvm/test/Transforms/InstSimplify/compare.ll @@ -1659,21 +1659,21 @@ define <2 x i1> @icmp_shl_1_ugt_signmask(<2 x i8> %V) { ret <2 x i1> %cmp } -define <2 x i1> @icmp_shl_1_ugt_signmask_undef(<2 x i8> %V) { -; CHECK-LABEL: @icmp_shl_1_ugt_signmask_undef( +define <2 x i1> @icmp_shl_1_ugt_signmask_poison(<2 x i8> %V) { +; CHECK-LABEL: @icmp_shl_1_ugt_signmask_poison( ; CHECK-NEXT: ret <2 x i1> zeroinitializer ; %shl = shl <2 x i8> , %V - %cmp = icmp ugt <2 x i8> %shl, + %cmp = icmp ugt <2 x i8> %shl, ret <2 x i1> %cmp } -define <2 x i1> @icmp_shl_1_ugt_signmask_undef2(<2 x i8> %V) { -; CHECK-LABEL: @icmp_shl_1_ugt_signmask_undef2( +define <2 x i1> @icmp_shl_1_ugt_signmask_poison2(<2 x i8> %V) { +; CHECK-LABEL: @icmp_shl_1_ugt_signmask_poison2( ; CHECK-NEXT: ret <2 x i1> zeroinitializer ; - %shl = shl <2 x i8> , %V - %cmp = icmp ugt <2 x i8> %shl, + %shl = shl <2 x i8> , %V + %cmp = icmp ugt <2 x i8> %shl, ret <2 x i1> %cmp } @@ -1695,21 +1695,21 @@ define <2 x i1> @icmp_shl_1_ule_signmask(<2 x i8> %V) { ret <2 x i1> %cmp } -define <2 x i1> @icmp_shl_1_ule_signmask_undef(<2 x i8> %V) { -; CHECK-LABEL: @icmp_shl_1_ule_signmask_undef( +define <2 x i1> @icmp_shl_1_ule_signmask_poison(<2 x i8> %V) { +; CHECK-LABEL: @icmp_shl_1_ule_signmask_poison( ; CHECK-NEXT: ret <2 x i1> ; %shl = shl <2 x i8> , %V - %cmp = icmp ule <2 x i8> %shl, + %cmp = icmp ule <2 x i8> %shl, ret <2 x i1> %cmp } -define <2 x i1> @icmp_shl_1_ule_signmask_undef2(<2 x i8> %V) { -; CHECK-LABEL: @icmp_shl_1_ule_signmask_undef2( +define <2 x i1> @icmp_shl_1_ule_signmask_poison2(<2 x i8> %V) { +; CHECK-LABEL: @icmp_shl_1_ule_signmask_poison2( ; CHECK-NEXT: ret <2 x i1> ; - %shl = shl <2 x i8> , %V - %cmp = icmp ule <2 x i8> %shl, + %shl = shl <2 x i8> , %V + %cmp = icmp ule <2 x i8> %shl, ret <2 x i1> %cmp } @@ -1731,12 +1731,12 @@ define <2 x i1> @shl_1_cmp_eq_nonpow2_splat(<2 x i32> %x) { ret <2 x i1> %c } -define <2 x i1> @shl_1_cmp_eq_nonpow2_splat_undef(<2 x i32> %x) { -; CHECK-LABEL: @shl_1_cmp_eq_nonpow2_splat_undef( +define <2 x i1> @shl_1_cmp_eq_nonpow2_splat_poison(<2 x i32> %x) { +; CHECK-LABEL: @shl_1_cmp_eq_nonpow2_splat_poison( ; CHECK-NEXT: ret <2 x i1> zeroinitializer ; %s = shl <2 x i32> , %x - %c = icmp eq <2 x i32> %s, + %c = icmp eq <2 x i32> %s, ret <2 x i1> %c } @@ -1758,12 +1758,12 @@ define <2 x i1> @shl_1_cmp_ne_nonpow2_splat(<2 x i32> %x) { ret <2 x i1> %c } -define <2 x i1> @shl_1_cmp_ne_nonpow2_splat_undef(<2 x i32> %x) { -; CHECK-LABEL: @shl_1_cmp_ne_nonpow2_splat_undef( +define <2 x i1> @shl_1_cmp_ne_nonpow2_splat_poison(<2 x i32> %x) { +; CHECK-LABEL: @shl_1_cmp_ne_nonpow2_splat_poison( ; CHECK-NEXT: ret <2 x i1> ; - %s = shl <2 x i32> , %x - %c = icmp ne <2 x i32> %s, + %s = shl <2 x i32> , %x + %c = icmp ne <2 x i32> %s, ret <2 x i1> %c } @@ -1776,12 +1776,12 @@ define i1 @shl_pow2_cmp_eq_nonpow2(i32 %x) { ret i1 %c } -define <2 x i1> @shl_pow21_cmp_ne_nonpow2_splat_undef(<2 x i32> %x) { -; CHECK-LABEL: @shl_pow21_cmp_ne_nonpow2_splat_undef( +define <2 x i1> @shl_pow21_cmp_ne_nonpow2_splat_poison(<2 x i32> %x) { +; CHECK-LABEL: @shl_pow21_cmp_ne_nonpow2_splat_poison( ; CHECK-NEXT: ret <2 x i1> ; - %s = shl <2 x i32> , %x - %c = icmp ne <2 x i32> %s, + %s = shl <2 x i32> , %x + %c = icmp ne <2 x i32> %s, ret <2 x i1> %c } @@ -1820,12 +1820,12 @@ define i1 @shl_pow2_cmp_eq_zero_nuw(i32 %x) { ret i1 %c } -define <2 x i1> @shl_pow2_cmp_ne_zero_nuw_splat_undef(<2 x i32> %x) { -; CHECK-LABEL: @shl_pow2_cmp_ne_zero_nuw_splat_undef( +define <2 x i1> @shl_pow2_cmp_ne_zero_nuw_splat_poison(<2 x i32> %x) { +; CHECK-LABEL: @shl_pow2_cmp_ne_zero_nuw_splat_poison( ; CHECK-NEXT: ret <2 x i1> ; - %s = shl nuw <2 x i32> , %x - %c = icmp ne <2 x i32> %s, + %s = shl nuw <2 x i32> , %x + %c = icmp ne <2 x i32> %s, ret <2 x i1> %c } @@ -1838,12 +1838,12 @@ define i1 @shl_pow2_cmp_ne_zero_nsw(i32 %x) { ret i1 %c } -define <2 x i1> @shl_pow2_cmp_eq_zero_nsw_splat_undef(<2 x i32> %x) { -; CHECK-LABEL: @shl_pow2_cmp_eq_zero_nsw_splat_undef( +define <2 x i1> @shl_pow2_cmp_eq_zero_nsw_splat_poison(<2 x i32> %x) { +; CHECK-LABEL: @shl_pow2_cmp_eq_zero_nsw_splat_poison( ; CHECK-NEXT: ret <2 x i1> zeroinitializer ; - %s = shl nsw <2 x i32> , %x - %c = icmp eq <2 x i32> %s, + %s = shl nsw <2 x i32> , %x + %c = icmp eq <2 x i32> %s, ret <2 x i1> %c } diff --git a/llvm/test/Transforms/InstSimplify/constantfold-add-nuw-allones-to-allones.ll b/llvm/test/Transforms/InstSimplify/constantfold-add-nuw-allones-to-allones.ll index 7c9d9a9e2c7c..92d6cc30d624 100644 --- a/llvm/test/Transforms/InstSimplify/constantfold-add-nuw-allones-to-allones.ll +++ b/llvm/test/Transforms/InstSimplify/constantfold-add-nuw-allones-to-allones.ll @@ -63,11 +63,11 @@ define <2 x i8> @add_vec(<2 x i8> %x) { ret <2 x i8> %ret } -define <3 x i8> @add_vec_undef(<3 x i8> %x) { -; CHECK-LABEL: @add_vec_undef( -; CHECK-NEXT: ret <3 x i8> +define <3 x i8> @add_vec_poison(<3 x i8> %x) { +; CHECK-LABEL: @add_vec_poison( +; CHECK-NEXT: ret <3 x i8> ; - %ret = add nuw <3 x i8> %x, + %ret = add nuw <3 x i8> %x, ret <3 x i8> %ret } diff --git a/llvm/test/Transforms/InstSimplify/constantfold-shl-nuw-C-to-C.ll b/llvm/test/Transforms/InstSimplify/constantfold-shl-nuw-C-to-C.ll index b5b5773fee53..3f4a08807a4b 100644 --- a/llvm/test/Transforms/InstSimplify/constantfold-shl-nuw-C-to-C.ll +++ b/llvm/test/Transforms/InstSimplify/constantfold-shl-nuw-C-to-C.ll @@ -78,11 +78,11 @@ define <2 x i8> @shl_vec(<2 x i8> %x) { ret <2 x i8> %ret } -define <3 x i8> @shl_vec_undef(<3 x i8> %x) { -; CHECK-LABEL: @shl_vec_undef( -; CHECK-NEXT: ret <3 x i8> +define <3 x i8> @shl_vec_poison(<3 x i8> %x) { +; CHECK-LABEL: @shl_vec_poison( +; CHECK-NEXT: ret <3 x i8> ; - %ret = shl nuw <3 x i8> , %x + %ret = shl nuw <3 x i8> , %x ret <3 x i8> %ret } diff --git a/llvm/test/Transforms/InstSimplify/div.ll b/llvm/test/Transforms/InstSimplify/div.ll index e13b6f139bcf..5ca2e8837b92 100644 --- a/llvm/test/Transforms/InstSimplify/div.ll +++ b/llvm/test/Transforms/InstSimplify/div.ll @@ -17,11 +17,11 @@ define <2 x i32> @zero_dividend_vector(<2 x i32> %A) { ret <2 x i32> %B } -define <2 x i32> @zero_dividend_vector_undef_elt(<2 x i32> %A) { -; CHECK-LABEL: @zero_dividend_vector_undef_elt( +define <2 x i32> @zero_dividend_vector_poison_elt(<2 x i32> %A) { +; CHECK-LABEL: @zero_dividend_vector_poison_elt( ; CHECK-NEXT: ret <2 x i32> zeroinitializer ; - %B = sdiv <2 x i32> , %A + %B = sdiv <2 x i32> , %A ret <2 x i32> %B } @@ -59,23 +59,23 @@ define <2 x i8> @udiv_zero_elt_vec(<2 x i8> %x) { ret <2 x i8> %div } -define <2 x i8> @sdiv_undef_elt_vec(<2 x i8> %x) { -; CHECK-LABEL: @sdiv_undef_elt_vec( +define <2 x i8> @sdiv_poison_elt_vec(<2 x i8> %x) { +; CHECK-LABEL: @sdiv_poison_elt_vec( ; CHECK-NEXT: ret <2 x i8> poison ; - %div = sdiv <2 x i8> %x, + %div = sdiv <2 x i8> %x, ret <2 x i8> %div } -define <2 x i8> @udiv_undef_elt_vec(<2 x i8> %x) { -; CHECK-LABEL: @udiv_undef_elt_vec( +define <2 x i8> @udiv_poison_elt_vec(<2 x i8> %x) { +; CHECK-LABEL: @udiv_poison_elt_vec( ; CHECK-NEXT: ret <2 x i8> poison ; - %div = udiv <2 x i8> %x, + %div = udiv <2 x i8> %x, ret <2 x i8> %div } -; Division-by-zero is undef. UB in any vector lane means the whole op is undef. +; Division-by-zero is poison. UB in any vector lane means the whole op is poison. ; Thus, we can simplify this: if any element of 'y' is 0, we can do anything. ; Therefore, assume that all elements of 'y' must be 1. diff --git a/llvm/test/Transforms/InstSimplify/fast-math-strictfp.ll b/llvm/test/Transforms/InstSimplify/fast-math-strictfp.ll index 4938987baccc..b1d772890aff 100644 --- a/llvm/test/Transforms/InstSimplify/fast-math-strictfp.ll +++ b/llvm/test/Transforms/InstSimplify/fast-math-strictfp.ll @@ -18,11 +18,11 @@ define float @mul_zero_2(float %a) #0 { ret float %b } -define <2 x float> @mul_zero_nsz_nnan_vec_undef(<2 x float> %a) #0 { -; CHECK-LABEL: @mul_zero_nsz_nnan_vec_undef( +define <2 x float> @mul_zero_nsz_nnan_vec_poison(<2 x float> %a) #0 { +; CHECK-LABEL: @mul_zero_nsz_nnan_vec_poison( ; CHECK-NEXT: ret <2 x float> zeroinitializer ; - %b = call nsz nnan <2 x float> @llvm.experimental.constrained.fmul.v2f32(<2 x float> %a, <2 x float>, metadata !"round.tonearest", metadata !"fpexcept.ignore") + %b = call nsz nnan <2 x float> @llvm.experimental.constrained.fmul.v2f32(<2 x float> %a, <2 x float>, metadata !"round.tonearest", metadata !"fpexcept.ignore") ret <2 x float> %b } @@ -98,13 +98,13 @@ define <2 x float> @fadd_unary_fnegx_commute_vec(<2 x float> %x) #0 { ret <2 x float> %r } -define <2 x float> @fadd_fnegx_commute_vec_undef(<2 x float> %x) #0 { -; CHECK-LABEL: @fadd_fnegx_commute_vec_undef( -; CHECK-NEXT: [[NEGX:%.*]] = call <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float> , <2 x float> [[X:%.*]], metadata !"round.tonearest", metadata !"fpexcept.ignore") +define <2 x float> @fadd_fnegx_commute_vec_poison(<2 x float> %x) #0 { +; CHECK-LABEL: @fadd_fnegx_commute_vec_poison( +; CHECK-NEXT: [[NEGX:%.*]] = call <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float> , <2 x float> [[X:%.*]], metadata !"round.tonearest", metadata !"fpexcept.ignore") ; CHECK-NEXT: [[R:%.*]] = call nnan <2 x float> @llvm.experimental.constrained.fadd.v2f32(<2 x float> [[X]], <2 x float> [[NEGX]], metadata !"round.tonearest", metadata !"fpexcept.ignore") ; CHECK-NEXT: ret <2 x float> [[R]] ; - %negx = call <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float> , <2 x float> %x, metadata !"round.tonearest", metadata !"fpexcept.ignore") + %negx = call <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float> , <2 x float> %x, metadata !"round.tonearest", metadata !"fpexcept.ignore") %r = call nnan <2 x float> @llvm.experimental.constrained.fadd.v2f32(<2 x float> %x, <2 x float> %negx, metadata !"round.tonearest", metadata !"fpexcept.ignore") ret <2 x float> %r } @@ -240,34 +240,34 @@ define float @fneg_x(float %a) #0 { ret float %ret } -define <2 x float> @fsub_0_0_x_vec_undef1(<2 x float> %a) #0 { -; CHECK-LABEL: @fsub_0_0_x_vec_undef1( -; CHECK-NEXT: [[T1:%.*]] = call <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float> , <2 x float> [[A:%.*]], metadata !"round.tonearest", metadata !"fpexcept.ignore") +define <2 x float> @fsub_0_0_x_vec_poison1(<2 x float> %a) #0 { +; CHECK-LABEL: @fsub_0_0_x_vec_poison1( +; CHECK-NEXT: [[T1:%.*]] = call <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float> , <2 x float> [[A:%.*]], metadata !"round.tonearest", metadata !"fpexcept.ignore") ; CHECK-NEXT: [[RET:%.*]] = call nsz <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float> zeroinitializer, <2 x float> [[T1]], metadata !"round.tonearest", metadata !"fpexcept.ignore") ; CHECK-NEXT: ret <2 x float> [[RET]] ; - %t1 = call <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float> , <2 x float> %a, metadata !"round.tonearest", metadata !"fpexcept.ignore") + %t1 = call <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float> , <2 x float> %a, metadata !"round.tonearest", metadata !"fpexcept.ignore") %ret = call nsz <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float> zeroinitializer, <2 x float> %t1, metadata !"round.tonearest", metadata !"fpexcept.ignore") ret <2 x float> %ret } -define <2 x float> @fneg_x_vec_undef1(<2 x float> %a) #0 { -; CHECK-LABEL: @fneg_x_vec_undef1( +define <2 x float> @fneg_x_vec_poison1(<2 x float> %a) #0 { +; CHECK-LABEL: @fneg_x_vec_poison1( ; CHECK-NEXT: ret <2 x float> [[A:%.*]] ; %t1 = fneg <2 x float> %a - %ret = call nsz <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float> , <2 x float> %t1, metadata !"round.tonearest", metadata !"fpexcept.ignore") + %ret = call nsz <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float> , <2 x float> %t1, metadata !"round.tonearest", metadata !"fpexcept.ignore") ret <2 x float> %ret } -define <2 x float> @fsub_0_0_x_vec_undef2(<2 x float> %a) #0 { -; CHECK-LABEL: @fsub_0_0_x_vec_undef2( +define <2 x float> @fsub_0_0_x_vec_poison2(<2 x float> %a) #0 { +; CHECK-LABEL: @fsub_0_0_x_vec_poison2( ; CHECK-NEXT: [[T1:%.*]] = call <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float> zeroinitializer, <2 x float> [[A:%.*]], metadata !"round.tonearest", metadata !"fpexcept.ignore") -; CHECK-NEXT: [[RET:%.*]] = call nsz <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float> , <2 x float> [[T1]], metadata !"round.tonearest", metadata !"fpexcept.ignore") +; CHECK-NEXT: [[RET:%.*]] = call nsz <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float> , <2 x float> [[T1]], metadata !"round.tonearest", metadata !"fpexcept.ignore") ; CHECK-NEXT: ret <2 x float> [[RET]] ; %t1 = call <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float> zeroinitializer, <2 x float> %a, metadata !"round.tonearest", metadata !"fpexcept.ignore") - %ret = call nsz <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float> , <2 x float> %t1, metadata !"round.tonearest", metadata !"fpexcept.ignore") + %ret = call nsz <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float> , <2 x float> %t1, metadata !"round.tonearest", metadata !"fpexcept.ignore") ret <2 x float> %ret } @@ -281,11 +281,11 @@ define <2 x float> @fadd_zero_nsz_vec(<2 x float> %x) #0 { ret <2 x float> %r } -define <2 x float> @fadd_zero_nsz_vec_undef(<2 x float> %x) #0 { -; CHECK-LABEL: @fadd_zero_nsz_vec_undef( +define <2 x float> @fadd_zero_nsz_vec_poison(<2 x float> %x) #0 { +; CHECK-LABEL: @fadd_zero_nsz_vec_poison( ; CHECK-NEXT: ret <2 x float> [[X:%.*]] ; - %r = call nsz <2 x float> @llvm.experimental.constrained.fadd.v2f32(<2 x float> %x, <2 x float> , metadata !"round.tonearest", metadata !"fpexcept.ignore") + %r = call nsz <2 x float> @llvm.experimental.constrained.fadd.v2f32(<2 x float> %x, <2 x float> , metadata !"round.tonearest", metadata !"fpexcept.ignore") ret <2 x float> %r } @@ -375,11 +375,11 @@ define double @fdiv_zero_by_x(double %x) #0 { ret double %r } -define <2 x double> @fdiv_zero_by_x_vec_undef(<2 x double> %x) #0 { -; CHECK-LABEL: @fdiv_zero_by_x_vec_undef( +define <2 x double> @fdiv_zero_by_x_vec_poison(<2 x double> %x) #0 { +; CHECK-LABEL: @fdiv_zero_by_x_vec_poison( ; CHECK-NEXT: ret <2 x double> zeroinitializer ; - %r = call nnan nsz <2 x double> @llvm.experimental.constrained.fdiv.v2f64(<2 x double> , <2 x double> %x, metadata !"round.tonearest", metadata !"fpexcept.ignore") + %r = call nnan nsz <2 x double> @llvm.experimental.constrained.fdiv.v2f64(<2 x double> , <2 x double> %x, metadata !"round.tonearest", metadata !"fpexcept.ignore") ret <2 x double> %r } @@ -394,11 +394,11 @@ define double @frem_zero_by_x(double %x) #0 { ret double %r } -define <2 x double> @frem_poszero_by_x_vec_undef(<2 x double> %x) #0 { -; CHECK-LABEL: @frem_poszero_by_x_vec_undef( +define <2 x double> @frem_poszero_by_x_vec_poison(<2 x double> %x) #0 { +; CHECK-LABEL: @frem_poszero_by_x_vec_poison( ; CHECK-NEXT: ret <2 x double> zeroinitializer ; - %r = call nnan <2 x double> @llvm.experimental.constrained.frem.v2f64(<2 x double> , <2 x double> %x, metadata !"round.tonearest", metadata !"fpexcept.ignore") + %r = call nnan <2 x double> @llvm.experimental.constrained.frem.v2f64(<2 x double> , <2 x double> %x, metadata !"round.tonearest", metadata !"fpexcept.ignore") ret <2 x double> %r } @@ -413,11 +413,11 @@ define double @frem_negzero_by_x(double %x) #0 { ret double %r } -define <2 x double> @frem_negzero_by_x_vec_undef(<2 x double> %x) #0 { -; CHECK-LABEL: @frem_negzero_by_x_vec_undef( +define <2 x double> @frem_negzero_by_x_vec_poison(<2 x double> %x) #0 { +; CHECK-LABEL: @frem_negzero_by_x_vec_poison( ; CHECK-NEXT: ret <2 x double> ; - %r = call nnan <2 x double> @llvm.experimental.constrained.frem.v2f64(<2 x double> , <2 x double> %x, metadata !"round.tonearest", metadata !"fpexcept.ignore") + %r = call nnan <2 x double> @llvm.experimental.constrained.frem.v2f64(<2 x double> , <2 x double> %x, metadata !"round.tonearest", metadata !"fpexcept.ignore") ret <2 x double> %r } @@ -493,13 +493,13 @@ define float @fdiv_neg_swapped2(float %f) #0 { ret float %div } -define <2 x float> @fdiv_neg_vec_undef_elt(<2 x float> %f) #0 { -; CHECK-LABEL: @fdiv_neg_vec_undef_elt( -; CHECK-NEXT: [[NEG:%.*]] = call <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float> , <2 x float> [[F:%.*]], metadata !"round.tonearest", metadata !"fpexcept.ignore") +define <2 x float> @fdiv_neg_vec_poison_elt(<2 x float> %f) #0 { +; CHECK-LABEL: @fdiv_neg_vec_poison_elt( +; CHECK-NEXT: [[NEG:%.*]] = call <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float> , <2 x float> [[F:%.*]], metadata !"round.tonearest", metadata !"fpexcept.ignore") ; CHECK-NEXT: [[DIV:%.*]] = call nnan <2 x float> @llvm.experimental.constrained.fdiv.v2f32(<2 x float> [[F]], <2 x float> [[NEG]], metadata !"round.tonearest", metadata !"fpexcept.ignore") ; CHECK-NEXT: ret <2 x float> [[DIV]] ; - %neg = call <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float> , <2 x float> %f, metadata !"round.tonearest", metadata !"fpexcept.ignore") + %neg = call <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float> , <2 x float> %f, metadata !"round.tonearest", metadata !"fpexcept.ignore") %div = call nnan <2 x float> @llvm.experimental.constrained.fdiv.v2f32(<2 x float> %f, <2 x float> %neg, metadata !"round.tonearest", metadata !"fpexcept.ignore") ret <2 x float> %div } diff --git a/llvm/test/Transforms/InstSimplify/fast-math.ll b/llvm/test/Transforms/InstSimplify/fast-math.ll index d1818e6346d7..287f30b162f8 100644 --- a/llvm/test/Transforms/InstSimplify/fast-math.ll +++ b/llvm/test/Transforms/InstSimplify/fast-math.ll @@ -18,11 +18,11 @@ define float @mul_zero_2(float %a) { ret float %b } -define <2 x float> @mul_zero_nsz_nnan_vec_undef(<2 x float> %a) { -; CHECK-LABEL: @mul_zero_nsz_nnan_vec_undef( +define <2 x float> @mul_zero_nsz_nnan_vec_poison(<2 x float> %a) { +; CHECK-LABEL: @mul_zero_nsz_nnan_vec_poison( ; CHECK-NEXT: ret <2 x float> zeroinitializer ; - %b = fmul nsz nnan <2 x float> %a, + %b = fmul nsz nnan <2 x float> %a, ret <2 x float> %b } @@ -94,11 +94,11 @@ define <2 x float> @fadd_unary_fnegx_commute_vec(<2 x float> %x) { ret <2 x float> %r } -define <2 x float> @fadd_fnegx_commute_vec_undef(<2 x float> %x) { -; CHECK-LABEL: @fadd_fnegx_commute_vec_undef( +define <2 x float> @fadd_fnegx_commute_vec_poison(<2 x float> %x) { +; CHECK-LABEL: @fadd_fnegx_commute_vec_poison( ; CHECK-NEXT: ret <2 x float> zeroinitializer ; - %negx = fsub <2 x float> , %x + %negx = fsub <2 x float> , %x %r = fadd nnan <2 x float> %x, %negx ret <2 x float> %r } @@ -226,30 +226,30 @@ define float @fneg_x(float %a) { ret float %ret } -define <2 x float> @fsub_0_0_x_vec_undef1(<2 x float> %a) { -; CHECK-LABEL: @fsub_0_0_x_vec_undef1( +define <2 x float> @fsub_0_0_x_vec_poison1(<2 x float> %a) { +; CHECK-LABEL: @fsub_0_0_x_vec_poison1( ; CHECK-NEXT: ret <2 x float> [[A:%.*]] ; - %t1 = fsub <2 x float> , %a + %t1 = fsub <2 x float> , %a %ret = fsub nsz <2 x float> zeroinitializer, %t1 ret <2 x float> %ret } -define <2 x float> @fneg_x_vec_undef1(<2 x float> %a) { -; CHECK-LABEL: @fneg_x_vec_undef1( +define <2 x float> @fneg_x_vec_poison1(<2 x float> %a) { +; CHECK-LABEL: @fneg_x_vec_poison1( ; CHECK-NEXT: ret <2 x float> [[A:%.*]] ; %t1 = fneg <2 x float> %a - %ret = fsub nsz <2 x float> , %t1 + %ret = fsub nsz <2 x float> , %t1 ret <2 x float> %ret } -define <2 x float> @fsub_0_0_x_vec_undef2(<2 x float> %a) { -; CHECK-LABEL: @fsub_0_0_x_vec_undef2( +define <2 x float> @fsub_0_0_x_vec_poison2(<2 x float> %a) { +; CHECK-LABEL: @fsub_0_0_x_vec_poison2( ; CHECK-NEXT: ret <2 x float> [[A:%.*]] ; %t1 = fsub <2 x float> zeroinitializer, %a - %ret = fsub nsz <2 x float> , %t1 + %ret = fsub nsz <2 x float> , %t1 ret <2 x float> %ret } @@ -263,11 +263,11 @@ define <2 x float> @fadd_zero_nsz_vec(<2 x float> %x) { ret <2 x float> %r } -define <2 x float> @fadd_zero_nsz_vec_undef(<2 x float> %x) { -; CHECK-LABEL: @fadd_zero_nsz_vec_undef( +define <2 x float> @fadd_zero_nsz_vec_poison(<2 x float> %x) { +; CHECK-LABEL: @fadd_zero_nsz_vec_poison( ; CHECK-NEXT: ret <2 x float> [[X:%.*]] ; - %r = fadd nsz <2 x float> %x, + %r = fadd nsz <2 x float> %x, ret <2 x float> %r } @@ -357,11 +357,11 @@ define double @fdiv_zero_by_x(double %x) { ret double %r } -define <2 x double> @fdiv_zero_by_x_vec_undef(<2 x double> %x) { -; CHECK-LABEL: @fdiv_zero_by_x_vec_undef( +define <2 x double> @fdiv_zero_by_x_vec_poison(<2 x double> %x) { +; CHECK-LABEL: @fdiv_zero_by_x_vec_poison( ; CHECK-NEXT: ret <2 x double> zeroinitializer ; - %r = fdiv nnan nsz <2 x double> , %x + %r = fdiv nnan nsz <2 x double> , %x ret <2 x double> %r } @@ -376,11 +376,11 @@ define double @frem_zero_by_x(double %x) { ret double %r } -define <2 x double> @frem_poszero_by_x_vec_undef(<2 x double> %x) { -; CHECK-LABEL: @frem_poszero_by_x_vec_undef( +define <2 x double> @frem_poszero_by_x_vec_poison(<2 x double> %x) { +; CHECK-LABEL: @frem_poszero_by_x_vec_poison( ; CHECK-NEXT: ret <2 x double> zeroinitializer ; - %r = frem nnan <2 x double> , %x + %r = frem nnan <2 x double> , %x ret <2 x double> %r } @@ -395,11 +395,11 @@ define double @frem_negzero_by_x(double %x) { ret double %r } -define <2 x double> @frem_negzero_by_x_vec_undef(<2 x double> %x) { -; CHECK-LABEL: @frem_negzero_by_x_vec_undef( +define <2 x double> @frem_negzero_by_x_vec_poison(<2 x double> %x) { +; CHECK-LABEL: @frem_negzero_by_x_vec_poison( ; CHECK-NEXT: ret <2 x double> ; - %r = frem nnan <2 x double> , %x + %r = frem nnan <2 x double> , %x ret <2 x double> %r } @@ -467,11 +467,11 @@ define float @fdiv_neg_swapped2(float %f) { ret float %div } -define <2 x float> @fdiv_neg_vec_undef_elt(<2 x float> %f) { -; CHECK-LABEL: @fdiv_neg_vec_undef_elt( +define <2 x float> @fdiv_neg_vec_poison_elt(<2 x float> %f) { +; CHECK-LABEL: @fdiv_neg_vec_poison_elt( ; CHECK-NEXT: ret <2 x float> ; - %neg = fsub <2 x float> , %f + %neg = fsub <2 x float> , %f %div = fdiv nnan <2 x float> %f, %neg ret <2 x float> %div } diff --git a/llvm/test/Transforms/InstSimplify/fdiv.ll b/llvm/test/Transforms/InstSimplify/fdiv.ll index 38e31257e185..fb59011b91d5 100644 --- a/llvm/test/Transforms/InstSimplify/fdiv.ll +++ b/llvm/test/Transforms/InstSimplify/fdiv.ll @@ -110,11 +110,11 @@ define <2 x float> @fdiv_nnan_ninf_by_undef_v2f32(<2 x float> %x) { ret <2 x float> %fdiv } -define <2 x float> @fdiv_nnan_ninf_by_zero_undef_v2f32(<2 x float> %x) { -; CHECK-LABEL: @fdiv_nnan_ninf_by_zero_undef_v2f32( +define <2 x float> @fdiv_nnan_ninf_by_zero_poison_v2f32(<2 x float> %x) { +; CHECK-LABEL: @fdiv_nnan_ninf_by_zero_poison_v2f32( ; CHECK-NEXT: ret <2 x float> poison ; - %fdiv = fdiv nnan ninf <2 x float> %x, + %fdiv = fdiv nnan ninf <2 x float> %x, ret <2 x float> %fdiv } diff --git a/llvm/test/Transforms/InstSimplify/floating-point-arithmetic-strictfp.ll b/llvm/test/Transforms/InstSimplify/floating-point-arithmetic-strictfp.ll index e4748a240292..32ea4cb7cd19 100644 --- a/llvm/test/Transforms/InstSimplify/floating-point-arithmetic-strictfp.ll +++ b/llvm/test/Transforms/InstSimplify/floating-point-arithmetic-strictfp.ll @@ -24,23 +24,23 @@ define <2 x float> @fsub_-0_x_vec(<2 x float> %a) #0 { ret <2 x float> %ret } -define <2 x float> @fsub_-0_x_vec_undef_elts(<2 x float> %a) #0 { -; CHECK-LABEL: @fsub_-0_x_vec_undef_elts( -; CHECK-NEXT: [[T1:%.*]] = call <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float> , <2 x float> [[A:%.*]], metadata !"round.tonearest", metadata !"fpexcept.ignore") +define <2 x float> @fsub_-0_x_vec_poison_elts(<2 x float> %a) #0 { +; CHECK-LABEL: @fsub_-0_x_vec_poison_elts( +; CHECK-NEXT: [[T1:%.*]] = call <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float> , <2 x float> [[A:%.*]], metadata !"round.tonearest", metadata !"fpexcept.ignore") ; CHECK-NEXT: [[RET:%.*]] = fneg <2 x float> [[T1]] ; CHECK-NEXT: ret <2 x float> [[RET]] ; - %t1 = call <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float>, <2 x float> %a, metadata !"round.tonearest", metadata !"fpexcept.ignore") + %t1 = call <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float>, <2 x float> %a, metadata !"round.tonearest", metadata !"fpexcept.ignore") %ret = fneg <2 x float> %t1 ret <2 x float> %ret } -define <2 x float> @fsub_negzero_vec_undef_elts(<2 x float> %x) #0 { -; CHECK-LABEL: @fsub_negzero_vec_undef_elts( -; CHECK-NEXT: [[R:%.*]] = call nsz <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float> , <2 x float> [[X:%.*]], metadata !"round.tonearest", metadata !"fpexcept.ignore") +define <2 x float> @fsub_negzero_vec_poison_elts(<2 x float> %x) #0 { +; CHECK-LABEL: @fsub_negzero_vec_poison_elts( +; CHECK-NEXT: [[R:%.*]] = call nsz <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float> , <2 x float> [[X:%.*]], metadata !"round.tonearest", metadata !"fpexcept.ignore") ; CHECK-NEXT: ret <2 x float> [[R]] ; - %r = call nsz <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float>, <2 x float> %x, metadata !"round.tonearest", metadata !"fpexcept.ignore") + %r = call nsz <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float>, <2 x float> %x, metadata !"round.tonearest", metadata !"fpexcept.ignore") ret <2 x float> %r } @@ -86,23 +86,23 @@ define <2 x float> @fneg_x_vec(<2 x float> %a) #0 { ret <2 x float> %ret } -define <2 x float> @fsub_-0_-0_x_vec_undef_elts(<2 x float> %a) #0 { -; CHECK-LABEL: @fsub_-0_-0_x_vec_undef_elts( -; CHECK-NEXT: [[T1:%.*]] = call <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float> , <2 x float> [[A:%.*]], metadata !"round.tonearest", metadata !"fpexcept.ignore") -; CHECK-NEXT: [[RET:%.*]] = call <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float> , <2 x float> [[T1]], metadata !"round.tonearest", metadata !"fpexcept.ignore") +define <2 x float> @fsub_-0_-0_x_vec_poison_elts(<2 x float> %a) #0 { +; CHECK-LABEL: @fsub_-0_-0_x_vec_poison_elts( +; CHECK-NEXT: [[T1:%.*]] = call <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float> , <2 x float> [[A:%.*]], metadata !"round.tonearest", metadata !"fpexcept.ignore") +; CHECK-NEXT: [[RET:%.*]] = call <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float> , <2 x float> [[T1]], metadata !"round.tonearest", metadata !"fpexcept.ignore") ; CHECK-NEXT: ret <2 x float> [[RET]] ; - %t1 = call <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float>, <2 x float> %a, metadata !"round.tonearest", metadata !"fpexcept.ignore") - %ret = call <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float>, <2 x float> %t1, metadata !"round.tonearest", metadata !"fpexcept.ignore") + %t1 = call <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float>, <2 x float> %a, metadata !"round.tonearest", metadata !"fpexcept.ignore") + %ret = call <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float>, <2 x float> %t1, metadata !"round.tonearest", metadata !"fpexcept.ignore") ret <2 x float> %ret } -define <2 x float> @fneg_x_vec_undef_elts(<2 x float> %a) #0 { -; CHECK-LABEL: @fneg_x_vec_undef_elts( +define <2 x float> @fneg_x_vec_poison_elts(<2 x float> %a) #0 { +; CHECK-LABEL: @fneg_x_vec_poison_elts( ; CHECK-NEXT: ret <2 x float> [[A:%.*]] ; %t1 = fneg <2 x float> %a - %ret = call <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float>, <2 x float> %t1, metadata !"round.tonearest", metadata !"fpexcept.ignore") + %ret = call <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float>, <2 x float> %t1, metadata !"round.tonearest", metadata !"fpexcept.ignore") ret <2 x float> %ret } @@ -139,11 +139,11 @@ define float @fsub_x_0(float %x) #0 { ret float %r } -define <2 x float> @fsub_x_0_vec_undef(<2 x float> %x) #0 { -; CHECK-LABEL: @fsub_x_0_vec_undef( +define <2 x float> @fsub_x_0_vec_poison(<2 x float> %x) #0 { +; CHECK-LABEL: @fsub_x_0_vec_poison( ; CHECK-NEXT: ret <2 x float> [[X:%.*]] ; - %r = call <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float> %x, <2 x float>, metadata !"round.tonearest", metadata !"fpexcept.ignore") + %r = call <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float> %x, <2 x float>, metadata !"round.tonearest", metadata !"fpexcept.ignore") ret <2 x float> %r } @@ -156,11 +156,11 @@ define float @fadd_x_n0(float %a) #0 { ret float %ret } -define <2 x float> @fadd_x_n0_vec_undef_elt(<2 x float> %a) #0 { -; CHECK-LABEL: @fadd_x_n0_vec_undef_elt( +define <2 x float> @fadd_x_n0_vec_poison_elt(<2 x float> %a) #0 { +; CHECK-LABEL: @fadd_x_n0_vec_poison_elt( ; CHECK-NEXT: ret <2 x float> [[A:%.*]] ; - %ret = call <2 x float> @llvm.experimental.constrained.fadd.v2f32(<2 x float> %a, <2 x float> , metadata !"round.tonearest", metadata !"fpexcept.ignore") + %ret = call <2 x float> @llvm.experimental.constrained.fadd.v2f32(<2 x float> %a, <2 x float> , metadata !"round.tonearest", metadata !"fpexcept.ignore") ret <2 x float> %ret } @@ -174,12 +174,12 @@ define float @fadd_x_p0(float %a) #0 { ret float %ret } -define <2 x float> @fadd_x_p0_vec_undef_elt(<2 x float> %a) #0 { -; CHECK-LABEL: @fadd_x_p0_vec_undef_elt( -; CHECK-NEXT: [[RET:%.*]] = call <2 x float> @llvm.experimental.constrained.fadd.v2f32(<2 x float> [[A:%.*]], <2 x float> , metadata !"round.tonearest", metadata !"fpexcept.ignore") +define <2 x float> @fadd_x_p0_vec_poison_elt(<2 x float> %a) #0 { +; CHECK-LABEL: @fadd_x_p0_vec_poison_elt( +; CHECK-NEXT: [[RET:%.*]] = call <2 x float> @llvm.experimental.constrained.fadd.v2f32(<2 x float> [[A:%.*]], <2 x float> , metadata !"round.tonearest", metadata !"fpexcept.ignore") ; CHECK-NEXT: ret <2 x float> [[RET]] ; - %ret = call <2 x float> @llvm.experimental.constrained.fadd.v2f32(<2 x float> %a, <2 x float> , metadata !"round.tonearest", metadata !"fpexcept.ignore") + %ret = call <2 x float> @llvm.experimental.constrained.fadd.v2f32(<2 x float> %a, <2 x float> , metadata !"round.tonearest", metadata !"fpexcept.ignore") ret <2 x float> %ret } diff --git a/llvm/test/Transforms/InstSimplify/floating-point-arithmetic.ll b/llvm/test/Transforms/InstSimplify/floating-point-arithmetic.ll index 5d17504c09df..7a35f09f03b9 100644 --- a/llvm/test/Transforms/InstSimplify/floating-point-arithmetic.ll +++ b/llvm/test/Transforms/InstSimplify/floating-point-arithmetic.ll @@ -30,20 +30,20 @@ define <2 x float> @fsub_-0_x_vec(<2 x float> %a) { ret <2 x float> %ret } -define <2 x float> @fsub_-0_x_vec_undef_elts(<2 x float> %a) { -; CHECK-LABEL: @fsub_-0_x_vec_undef_elts( +define <2 x float> @fsub_-0_x_vec_poison_elts(<2 x float> %a) { +; CHECK-LABEL: @fsub_-0_x_vec_poison_elts( ; CHECK-NEXT: ret <2 x float> [[A:%.*]] ; - %t1 = fsub <2 x float> , %a + %t1 = fsub <2 x float> , %a %ret = fneg <2 x float> %t1 ret <2 x float> %ret } -define <2 x float> @fsub_negzero_vec_undef_elts(<2 x float> %x) { -; CHECK-LABEL: @fsub_negzero_vec_undef_elts( +define <2 x float> @fsub_negzero_vec_poison_elts(<2 x float> %x) { +; CHECK-LABEL: @fsub_negzero_vec_poison_elts( ; CHECK-NEXT: ret <2 x float> [[X:%.*]] ; - %r = fsub nsz <2 x float> %x, + %r = fsub nsz <2 x float> %x, ret <2 x float> %r } @@ -85,21 +85,21 @@ define <2 x float> @fneg_x_vec(<2 x float> %a) { ret <2 x float> %ret } -define <2 x float> @fsub_-0_-0_x_vec_undef_elts(<2 x float> %a) { -; CHECK-LABEL: @fsub_-0_-0_x_vec_undef_elts( +define <2 x float> @fsub_-0_-0_x_vec_poison_elts(<2 x float> %a) { +; CHECK-LABEL: @fsub_-0_-0_x_vec_poison_elts( ; CHECK-NEXT: ret <2 x float> [[A:%.*]] ; - %t1 = fsub <2 x float> , %a - %ret = fsub <2 x float> , %t1 + %t1 = fsub <2 x float> , %a + %ret = fsub <2 x float> , %t1 ret <2 x float> %ret } -define <2 x float> @fneg_x_vec_undef_elts(<2 x float> %a) { -; CHECK-LABEL: @fneg_x_vec_undef_elts( +define <2 x float> @fneg_x_vec_poison_elts(<2 x float> %a) { +; CHECK-LABEL: @fneg_x_vec_poison_elts( ; CHECK-NEXT: ret <2 x float> [[A:%.*]] ; %t1 = fneg <2 x float> %a - %ret = fsub <2 x float> , %t1 + %ret = fsub <2 x float> , %t1 ret <2 x float> %ret } @@ -136,11 +136,11 @@ define float @fsub_x_0(float %x) { ret float %r } -define <2 x float> @fsub_x_0_vec_undef(<2 x float> %x) { -; CHECK-LABEL: @fsub_x_0_vec_undef( +define <2 x float> @fsub_x_0_vec_poison(<2 x float> %x) { +; CHECK-LABEL: @fsub_x_0_vec_poison( ; CHECK-NEXT: ret <2 x float> [[X:%.*]] ; - %r = fsub <2 x float> %x, + %r = fsub <2 x float> %x, ret <2 x float> %r } @@ -153,11 +153,11 @@ define float @fadd_x_n0(float %a) { ret float %ret } -define <2 x float> @fadd_x_n0_vec_undef_elt(<2 x float> %a) { -; CHECK-LABEL: @fadd_x_n0_vec_undef_elt( +define <2 x float> @fadd_x_n0_vec_poison_elt(<2 x float> %a) { +; CHECK-LABEL: @fadd_x_n0_vec_poison_elt( ; CHECK-NEXT: ret <2 x float> [[A:%.*]] ; - %ret = fadd <2 x float> %a, + %ret = fadd <2 x float> %a, ret <2 x float> %ret } diff --git a/llvm/test/Transforms/InstSimplify/floating-point-compare.ll b/llvm/test/Transforms/InstSimplify/floating-point-compare.ll index 3c1794c81284..70f0321039ea 100644 --- a/llvm/test/Transforms/InstSimplify/floating-point-compare.ll +++ b/llvm/test/Transforms/InstSimplify/floating-point-compare.ll @@ -547,30 +547,30 @@ define <2 x i1> @fabs_is_not_negative_anyzero(<2 x float> %V) { ret <2 x i1> %cmp } -define <3 x i1> @fabs_is_not_negative_negzero_undef(<3 x float> %V) { -; CHECK-LABEL: @fabs_is_not_negative_negzero_undef( +define <3 x i1> @fabs_is_not_negative_negzero_poison(<3 x float> %V) { +; CHECK-LABEL: @fabs_is_not_negative_negzero_poison( ; CHECK-NEXT: ret <3 x i1> zeroinitializer ; %abs = call <3 x float> @llvm.fabs.v3f32(<3 x float> %V) - %cmp = fcmp olt <3 x float> %abs, + %cmp = fcmp olt <3 x float> %abs, ret <3 x i1> %cmp } -define <3 x i1> @fabs_is_not_negative_poszero_undef(<3 x float> %V) { -; CHECK-LABEL: @fabs_is_not_negative_poszero_undef( +define <3 x i1> @fabs_is_not_negative_poszero_poison(<3 x float> %V) { +; CHECK-LABEL: @fabs_is_not_negative_poszero_poison( ; CHECK-NEXT: ret <3 x i1> zeroinitializer ; %abs = call <3 x float> @llvm.fabs.v3f32(<3 x float> %V) - %cmp = fcmp olt <3 x float> %abs, + %cmp = fcmp olt <3 x float> %abs, ret <3 x i1> %cmp } -define <3 x i1> @fabs_is_not_negative_anyzero_undef(<3 x float> %V) { -; CHECK-LABEL: @fabs_is_not_negative_anyzero_undef( +define <3 x i1> @fabs_is_not_negative_anyzero_poison(<3 x float> %V) { +; CHECK-LABEL: @fabs_is_not_negative_anyzero_poison( ; CHECK-NEXT: ret <3 x i1> zeroinitializer ; %abs = call <3 x float> @llvm.fabs.v3f32(<3 x float> %V) - %cmp = fcmp olt <3 x float> %abs, + %cmp = fcmp olt <3 x float> %abs, ret <3 x i1> %cmp } @@ -1335,19 +1335,19 @@ define <2 x i1> @orderedCompareWithNaNVector(<2 x double> %A) { ret <2 x i1> %cmp } -define <2 x i1> @orderedCompareWithNaNVector_undef_elt(<2 x double> %A) { -; CHECK-LABEL: @orderedCompareWithNaNVector_undef_elt( +define <2 x i1> @orderedCompareWithNaNVector_poison_elt(<2 x double> %A) { +; CHECK-LABEL: @orderedCompareWithNaNVector_poison_elt( ; CHECK-NEXT: ret <2 x i1> zeroinitializer ; - %cmp = fcmp olt <2 x double> %A, + %cmp = fcmp olt <2 x double> %A, ret <2 x i1> %cmp } -define <2 x i1> @unorderedCompareWithNaNVector_undef_elt(<2 x double> %A) { -; CHECK-LABEL: @unorderedCompareWithNaNVector_undef_elt( +define <2 x i1> @unorderedCompareWithNaNVector_poison_elt(<2 x double> %A) { +; CHECK-LABEL: @unorderedCompareWithNaNVector_poison_elt( ; CHECK-NEXT: ret <2 x i1> ; - %cmp = fcmp ult <2 x double> %A, + %cmp = fcmp ult <2 x double> %A, ret <2 x i1> %cmp } diff --git a/llvm/test/Transforms/InstSimplify/fminmax-folds.ll b/llvm/test/Transforms/InstSimplify/fminmax-folds.ll index a8a9e96a652f..668a93ddf5a4 100644 --- a/llvm/test/Transforms/InstSimplify/fminmax-folds.ll +++ b/llvm/test/Transforms/InstSimplify/fminmax-folds.ll @@ -493,7 +493,7 @@ define <2 x double> @maxnum_nan_op0_vec(<2 x double> %x) { ; CHECK-LABEL: @maxnum_nan_op0_vec( ; CHECK-NEXT: ret <2 x double> [[X:%.*]] ; - %r = call <2 x double> @llvm.maxnum.v2f64(<2 x double> , <2 x double> %x) + %r = call <2 x double> @llvm.maxnum.v2f64(<2 x double> , <2 x double> %x) ret <2 x double> %r } @@ -509,7 +509,7 @@ define <2 x double> @minnum_nan_op0_vec(<2 x double> %x) { ; CHECK-LABEL: @minnum_nan_op0_vec( ; CHECK-NEXT: ret <2 x double> [[X:%.*]] ; - %r = call <2 x double> @llvm.minnum.v2f64(<2 x double> , <2 x double> %x) + %r = call <2 x double> @llvm.minnum.v2f64(<2 x double> , <2 x double> %x) ret <2 x double> %r } @@ -873,19 +873,19 @@ define double @minimum_nan_op1(double %x) { ret double %r } -define <2 x double> @maximum_nan_op0_vec_partial_undef(<2 x double> %x) { -; CHECK-LABEL: @maximum_nan_op0_vec_partial_undef( -; CHECK-NEXT: ret <2 x double> +define <2 x double> @maximum_nan_op0_vec_partial_poison(<2 x double> %x) { +; CHECK-LABEL: @maximum_nan_op0_vec_partial_poison( +; CHECK-NEXT: ret <2 x double> ; - %r = call <2 x double> @llvm.maximum.v2f64(<2 x double> , <2 x double> %x) + %r = call <2 x double> @llvm.maximum.v2f64(<2 x double> , <2 x double> %x) ret <2 x double> %r } -define <2 x double> @maximum_nan_op1_vec_partial_undef(<2 x double> %x) { -; CHECK-LABEL: @maximum_nan_op1_vec_partial_undef( -; CHECK-NEXT: ret <2 x double> +define <2 x double> @maximum_nan_op1_vec_partial_poison(<2 x double> %x) { +; CHECK-LABEL: @maximum_nan_op1_vec_partial_poison( +; CHECK-NEXT: ret <2 x double> ; - %r = call <2 x double> @llvm.maximum.v2f64(<2 x double> %x, <2 x double> ) + %r = call <2 x double> @llvm.maximum.v2f64(<2 x double> %x, <2 x double> ) ret <2 x double> %r } @@ -897,19 +897,19 @@ define <2 x double> @maximum_nan_op1_vec(<2 x double> %x) { ret <2 x double> %r } -define <2 x double> @minimum_nan_op0_vec_partial_undef(<2 x double> %x) { -; CHECK-LABEL: @minimum_nan_op0_vec_partial_undef( -; CHECK-NEXT: ret <2 x double> +define <2 x double> @minimum_nan_op0_vec_partial_poison(<2 x double> %x) { +; CHECK-LABEL: @minimum_nan_op0_vec_partial_poison( +; CHECK-NEXT: ret <2 x double> ; - %r = call <2 x double> @llvm.minimum.v2f64(<2 x double> , <2 x double> %x) + %r = call <2 x double> @llvm.minimum.v2f64(<2 x double> , <2 x double> %x) ret <2 x double> %r } -define <2 x double> @minimum_nan_op1_vec_partial_undef(<2 x double> %x) { -; CHECK-LABEL: @minimum_nan_op1_vec_partial_undef( -; CHECK-NEXT: ret <2 x double> +define <2 x double> @minimum_nan_op1_vec_partial_poison(<2 x double> %x) { +; CHECK-LABEL: @minimum_nan_op1_vec_partial_poison( +; CHECK-NEXT: ret <2 x double> ; - %r = call <2 x double> @llvm.minimum.v2f64(<2 x double> %x, <2 x double> ) + %r = call <2 x double> @llvm.minimum.v2f64(<2 x double> %x, <2 x double> ) ret <2 x double> %r } diff --git a/llvm/test/Transforms/InstSimplify/fp-nan.ll b/llvm/test/Transforms/InstSimplify/fp-nan.ll index cb0bed379078..bb557500822c 100644 --- a/llvm/test/Transforms/InstSimplify/fp-nan.ll +++ b/llvm/test/Transforms/InstSimplify/fp-nan.ll @@ -163,13 +163,13 @@ define <2 x double> @fsub_nan_poison_op1(<2 x double> %x) { ret <2 x double> %r } -; Vector with undef element +; Vector with poison element -define <2 x double> @frem_nan_undef_op0(<2 x double> %x) { -; CHECK-LABEL: @frem_nan_undef_op0( -; CHECK-NEXT: ret <2 x double> +define <2 x double> @frem_nan_poison_op0(<2 x double> %x) { +; CHECK-LABEL: @frem_nan_poison_op0( +; CHECK-NEXT: ret <2 x double> ; - %r = frem <2 x double> , %x + %r = frem <2 x double> , %x ret <2 x double> %r } @@ -177,7 +177,8 @@ define <2 x double> @frem_nan_undef_op0(<2 x double> %x) { define <3 x double> @fadd_nan_poison_undef_op1(<3 x double> %x) { ; CHECK-LABEL: @fadd_nan_poison_undef_op1( -; CHECK-NEXT: ret <3 x double> +; CHECK-NEXT: [[R:%.*]] = fadd <3 x double> [[X:%.*]], +; CHECK-NEXT: ret <3 x double> [[R]] ; %r = fadd <3 x double> %x, ret <3 x double> %r diff --git a/llvm/test/Transforms/InstSimplify/icmp-bool-constant.ll b/llvm/test/Transforms/InstSimplify/icmp-bool-constant.ll index 6205225098a7..a501f995b6c9 100644 --- a/llvm/test/Transforms/InstSimplify/icmp-bool-constant.ll +++ b/llvm/test/Transforms/InstSimplify/icmp-bool-constant.ll @@ -12,11 +12,11 @@ define <2 x i1> @eq_t(<2 x i1> %a) { ret <2 x i1> %r } -define <2 x i1> @eq_t_undef_elt(<2 x i1> %a) { -; CHECK-LABEL: @eq_t_undef_elt( +define <2 x i1> @eq_t_poison_elt(<2 x i1> %a) { +; CHECK-LABEL: @eq_t_poison_elt( ; CHECK-NEXT: ret <2 x i1> [[A:%.*]] ; - %r = icmp eq <2 x i1> %a, + %r = icmp eq <2 x i1> %a, ret <2 x i1> %r } @@ -54,11 +54,11 @@ define <2 x i1> @ugt_t(<2 x i1> %a) { ret <2 x i1> %r } -define <2 x i1> @ugt_t_undef_elt(<2 x i1> %a) { -; CHECK-LABEL: @ugt_t_undef_elt( +define <2 x i1> @ugt_t_poison_elt(<2 x i1> %a) { +; CHECK-LABEL: @ugt_t_poison_elt( ; CHECK-NEXT: ret <2 x i1> zeroinitializer ; - %r = icmp ugt <2 x i1> %a, + %r = icmp ugt <2 x i1> %a, ret <2 x i1> %r } @@ -161,11 +161,11 @@ define <2 x i1> @sge_t(<2 x i1> %a) { ret <2 x i1> %r } -define <2 x i1> @sge_t_undef_elt(<2 x i1> %a) { -; CHECK-LABEL: @sge_t_undef_elt( +define <2 x i1> @sge_t_poison_elt(<2 x i1> %a) { +; CHECK-LABEL: @sge_t_poison_elt( ; CHECK-NEXT: ret <2 x i1> ; - %r = icmp sge <2 x i1> %a, + %r = icmp sge <2 x i1> %a, ret <2 x i1> %r } diff --git a/llvm/test/Transforms/InstSimplify/icmp-not-bool-constant.ll b/llvm/test/Transforms/InstSimplify/icmp-not-bool-constant.ll index f4a0b6ddf662..045d773bf328 100644 --- a/llvm/test/Transforms/InstSimplify/icmp-not-bool-constant.ll +++ b/llvm/test/Transforms/InstSimplify/icmp-not-bool-constant.ll @@ -33,11 +33,11 @@ define <2 x i1> @eq_f_not_swap(<2 x i1> %a) { ret <2 x i1> %r } -define <2 x i1> @eq_f_not_undef(<2 x i1> %a) { -; CHECK-LABEL: @eq_f_not_undef( +define <2 x i1> @eq_f_not_poison(<2 x i1> %a) { +; CHECK-LABEL: @eq_f_not_poison( ; CHECK-NEXT: ret <2 x i1> [[A:%.*]] ; - %not = xor <2 x i1> %a, + %not = xor <2 x i1> %a, %r = icmp eq <2 x i1> %not, ret <2 x i1> %r } @@ -60,11 +60,11 @@ define <2 x i1> @ne_t_not_swap(<2 x i1> %a) { ret <2 x i1> %r } -define <2 x i1> @ne_t_not_undef(<2 x i1> %a) { -; CHECK-LABEL: @ne_t_not_undef( +define <2 x i1> @ne_t_not_poison(<2 x i1> %a) { +; CHECK-LABEL: @ne_t_not_poison( ; CHECK-NEXT: ret <2 x i1> [[A:%.*]] ; - %not = xor <2 x i1> %a, + %not = xor <2 x i1> %a, %r = icmp ne <2 x i1> %not, ret <2 x i1> %r } @@ -116,11 +116,11 @@ define <2 x i1> @ult_t_not_swap(<2 x i1> %a) { ret <2 x i1> %r } -define <2 x i1> @ult_t_not_undef(<2 x i1> %a) { -; CHECK-LABEL: @ult_t_not_undef( +define <2 x i1> @ult_t_not_poison(<2 x i1> %a) { +; CHECK-LABEL: @ult_t_not_poison( ; CHECK-NEXT: ret <2 x i1> [[A:%.*]] ; - %not = xor <2 x i1> %a, + %not = xor <2 x i1> %a, %r = icmp ult <2 x i1> %not, ret <2 x i1> %r } @@ -152,11 +152,11 @@ define <2 x i1> @sgt_t_not_swap(<2 x i1> %a) { ret <2 x i1> %r } -define <2 x i1> @sgt_t_not_undef(<2 x i1> %a) { -; CHECK-LABEL: @sgt_t_not_undef( +define <2 x i1> @sgt_t_not_poison(<2 x i1> %a) { +; CHECK-LABEL: @sgt_t_not_poison( ; CHECK-NEXT: ret <2 x i1> [[A:%.*]] ; - %not = xor <2 x i1> %a, + %not = xor <2 x i1> %a, %r = icmp sgt <2 x i1> %not, ret <2 x i1> %r } @@ -235,11 +235,11 @@ define <2 x i1> @ule_f_not_swap(<2 x i1> %a) { ret <2 x i1> %r } -define <2 x i1> @ule_f_not_undef(<2 x i1> %a) { -; CHECK-LABEL: @ule_f_not_undef( +define <2 x i1> @ule_f_not_poison(<2 x i1> %a) { +; CHECK-LABEL: @ule_f_not_poison( ; CHECK-NEXT: ret <2 x i1> [[A:%.*]] ; - %not = xor <2 x i1> %a, + %not = xor <2 x i1> %a, %r = icmp ule <2 x i1> %not, ret <2 x i1> %r } @@ -271,11 +271,11 @@ define <2 x i1> @sge_f_not_swap(<2 x i1> %a) { ret <2 x i1> %r } -define <2 x i1> @sge_f_not_undef(<2 x i1> %a) { -; CHECK-LABEL: @sge_f_not_undef( +define <2 x i1> @sge_f_not_poison(<2 x i1> %a) { +; CHECK-LABEL: @sge_f_not_poison( ; CHECK-NEXT: ret <2 x i1> [[A:%.*]] ; - %not = xor <2 x i1> %a, + %not = xor <2 x i1> %a, %r = icmp sge <2 x i1> %not, ret <2 x i1> %r } diff --git a/llvm/test/Transforms/InstSimplify/ldexp.ll b/llvm/test/Transforms/InstSimplify/ldexp.ll index c6bb0141199f..d39f6a1e4967 100644 --- a/llvm/test/Transforms/InstSimplify/ldexp.ll +++ b/llvm/test/Transforms/InstSimplify/ldexp.ll @@ -57,11 +57,12 @@ define void @ldexp_f32_exp0(float %x) { define void @ldexp_v2f32_exp0(<2 x float> %x) { ; CHECK-LABEL: @ldexp_v2f32_exp0( ; CHECK-NEXT: store volatile <2 x float> [[X:%.*]], ptr addrspace(1) undef, align 8 -; CHECK-NEXT: store volatile <2 x float> [[X]], ptr addrspace(1) undef, align 8 +; CHECK-NEXT: [[PART_UNDEF1:%.*]] = call <2 x float> @llvm.ldexp.v2f32.v2i32(<2 x float> [[X]], <2 x i32> ) +; CHECK-NEXT: store volatile <2 x float> [[PART_UNDEF1]], ptr addrspace(1) undef, align 8 ; CHECK-NEXT: store volatile <2 x float> [[X]], ptr addrspace(1) undef, align 8 ; CHECK-NEXT: ret void ; - %part.undef0 = call <2 x float> @llvm.ldexp.v2f32.v2i32(<2 x float> %x, <2 x i32> ) + %part.undef0 = call <2 x float> @llvm.ldexp.v2f32.v2i32(<2 x float> %x, <2 x i32> ) store volatile <2 x float> %part.undef0, ptr addrspace(1) undef %part.undef1 = call <2 x float> @llvm.ldexp.v2f32.v2i32(<2 x float> %x, <2 x i32> ) diff --git a/llvm/test/Transforms/InstSimplify/mul.ll b/llvm/test/Transforms/InstSimplify/mul.ll index 8ae7f1eaac92..a1b03a30fe4f 100644 --- a/llvm/test/Transforms/InstSimplify/mul.ll +++ b/llvm/test/Transforms/InstSimplify/mul.ll @@ -34,11 +34,11 @@ define <16 x i8> @mul_by_0_vec(<16 x i8> %a) { ret <16 x i8> %b } -define <2 x i8> @mul_by_0_vec_undef_elt(<2 x i8> %a) { -; CHECK-LABEL: @mul_by_0_vec_undef_elt( +define <2 x i8> @mul_by_0_vec_poison_elt(<2 x i8> %a) { +; CHECK-LABEL: @mul_by_0_vec_poison_elt( ; CHECK-NEXT: ret <2 x i8> zeroinitializer ; - %b = mul <2 x i8> %a, + %b = mul <2 x i8> %a, ret <2 x i8> %b } diff --git a/llvm/test/Transforms/InstSimplify/negate.ll b/llvm/test/Transforms/InstSimplify/negate.ll index d72a0db6d445..d07029becd1f 100644 --- a/llvm/test/Transforms/InstSimplify/negate.ll +++ b/llvm/test/Transforms/InstSimplify/negate.ll @@ -17,11 +17,11 @@ define <2 x i32> @negate_nuw_vec(<2 x i32> %x) { ret <2 x i32> %neg } -define <2 x i32> @negate_nuw_vec_undef_elt(<2 x i32> %x) { -; CHECK-LABEL: @negate_nuw_vec_undef_elt( +define <2 x i32> @negate_nuw_vec_poison_elt(<2 x i32> %x) { +; CHECK-LABEL: @negate_nuw_vec_poison_elt( ; CHECK-NEXT: ret <2 x i32> zeroinitializer ; - %neg = sub nuw <2 x i32> , %x + %neg = sub nuw <2 x i32> , %x ret <2 x i32> %neg } @@ -43,12 +43,12 @@ define <2 x i8> @negate_zero_or_minsigned_nsw_vec(<2 x i8> %x) { ret <2 x i8> %neg } -define <2 x i8> @negate_zero_or_minsigned_nsw_vec_undef_elt(<2 x i8> %x) { -; CHECK-LABEL: @negate_zero_or_minsigned_nsw_vec_undef_elt( +define <2 x i8> @negate_zero_or_minsigned_nsw_vec_poison_elt(<2 x i8> %x) { +; CHECK-LABEL: @negate_zero_or_minsigned_nsw_vec_poison_elt( ; CHECK-NEXT: ret <2 x i8> zeroinitializer ; %signbit = shl <2 x i8> %x, - %neg = sub nsw <2 x i8> , %signbit + %neg = sub nsw <2 x i8> , %signbit ret <2 x i8> %neg } diff --git a/llvm/test/Transforms/InstSimplify/or.ll b/llvm/test/Transforms/InstSimplify/or.ll index 913b760dd331..f241c6987b9e 100644 --- a/llvm/test/Transforms/InstSimplify/or.ll +++ b/llvm/test/Transforms/InstSimplify/or.ll @@ -17,11 +17,11 @@ define i32 @all_ones(i32 %A) { ret i32 %B } -define <3 x i8> @all_ones_vec_with_undef_elt(<3 x i8> %A) { -; CHECK-LABEL: @all_ones_vec_with_undef_elt( +define <3 x i8> @all_ones_vec_with_poison_elt(<3 x i8> %A) { +; CHECK-LABEL: @all_ones_vec_with_poison_elt( ; CHECK-NEXT: ret <3 x i8> ; - %B = or <3 x i8> %A, + %B = or <3 x i8> %A, ret <3 x i8> %B } @@ -68,11 +68,11 @@ define i32 @or_not(i32 %A) { ret i32 %B } -define <2 x i4> @or_not_commute_vec_undef(<2 x i4> %A) { -; CHECK-LABEL: @or_not_commute_vec_undef( +define <2 x i4> @or_not_commute_vec_poison(<2 x i4> %A) { +; CHECK-LABEL: @or_not_commute_vec_poison( ; CHECK-NEXT: ret <2 x i4> ; - %NotA = xor <2 x i4> %A, + %NotA = xor <2 x i4> %A, %B = or <2 x i4> %NotA, %A ret <2 x i4> %B } @@ -335,7 +335,7 @@ define <2 x i1> @or_with_not_op_commute4(<2 x i1> %a, <2 x i1> %b) { ; CHECK-NEXT: ret <2 x i1> ; %ab = and <2 x i1> %b, %a - %not = xor <2 x i1> %ab, + %not = xor <2 x i1> %ab, %r = or <2 x i1> %not, %a ret <2 x i1> %r } @@ -515,6 +515,21 @@ define <2 x i4> @and_or_not_or_commute7_undef_elt(<2 x i4> %A, <2 x i4> %B) { ret <2 x i4> %r } +; doing the same with poison is safe. + +define <2 x i4> @and_or_not_or_commute7_poison_elt(<2 x i4> %A, <2 x i4> %B) { +; CHECK-LABEL: @and_or_not_or_commute7_poison_elt( +; CHECK-NEXT: [[NOTA:%.*]] = xor <2 x i4> [[A:%.*]], +; CHECK-NEXT: ret <2 x i4> [[NOTA]] +; + %nota = xor <2 x i4> %A, + %and = and <2 x i4> %B, %nota + %or = or <2 x i4> %B, %A + %notab = xor <2 x i4> %or, + %r = or <2 x i4> %notab, %and + ret <2 x i4> %r +} + ; (A | B) | (A ^ B) --> A | B define i69 @or_or_xor(i69 %A, i69 %B) { @@ -769,6 +784,21 @@ define <2 x i4> @or_nxor_and_undef_elt(<2 x i4> %a, <2 x i4> %b) { ret <2 x i4> %r } +; Same with poison is safe. + +define <2 x i4> @or_nxor_and_poison_elt(<2 x i4> %a, <2 x i4> %b) { +; CHECK-LABEL: @or_nxor_and_poison_elt( +; CHECK-NEXT: [[XOR:%.*]] = xor <2 x i4> [[A:%.*]], [[B:%.*]] +; CHECK-NEXT: [[NOT:%.*]] = xor <2 x i4> [[XOR]], +; CHECK-NEXT: ret <2 x i4> [[NOT]] +; + %and = and <2 x i4> %b, %a + %xor = xor <2 x i4> %a, %b + %not = xor <2 x i4> %xor, + %r = or <2 x i4> %not, %and + ret <2 x i4> %r +} + ; ~(A ^ B) | (A | B) --> -1 define i4 @or_nxor_or_commute0(i4 %a, i4 %b) { @@ -849,15 +879,15 @@ define i4 @or_nxor_or_wrong_val2(i4 %a, i4 %b, i4 %c) { ret i4 %r } -; negative test - undef in 'not' is allowed +; negative test - poison in 'not' is allowed -define <2 x i4> @or_nxor_or_undef_elt(<2 x i4> %a, <2 x i4> %b) { -; CHECK-LABEL: @or_nxor_or_undef_elt( +define <2 x i4> @or_nxor_or_poison_elt(<2 x i4> %a, <2 x i4> %b) { +; CHECK-LABEL: @or_nxor_or_poison_elt( ; CHECK-NEXT: ret <2 x i4> ; %or = or <2 x i4> %b, %a %xor = xor <2 x i4> %a, %b - %not = xor <2 x i4> %xor, + %not = xor <2 x i4> %xor, %r = or <2 x i4> %or, %not ret <2 x i4> %r } @@ -966,12 +996,12 @@ define i32 @or_xor_not_op_or_commute7(i32 %a, i32 %b){ ret i32 %r } -define <2 x i4> @or_xor_not_op_or_undef_elt(<2 x i4> %a, <2 x i4> %b) { -; CHECK-LABEL: @or_xor_not_op_or_undef_elt( +define <2 x i4> @or_xor_not_op_or_poison_elt(<2 x i4> %a, <2 x i4> %b) { +; CHECK-LABEL: @or_xor_not_op_or_poison_elt( ; CHECK-NEXT: ret <2 x i4> ; %xor = xor <2 x i4> %a, %b - %nota = xor <2 x i4> %a, + %nota = xor <2 x i4> %a, %or = or <2 x i4> %nota, %b %r = or <2 x i4> %xor, %or ret <2 x i4> %r @@ -1082,6 +1112,21 @@ define <2 x i4> @or_nand_xor_undef_elt(<2 x i4> %x, <2 x i4> %y) { ret <2 x i4> %or } +; Same with poison is safe. + +define <2 x i4> @or_nand_xor_poison_elt(<2 x i4> %x, <2 x i4> %y) { +; CHECK-LABEL: @or_nand_xor_poison_elt( +; CHECK-NEXT: [[AND:%.*]] = and <2 x i4> [[Y:%.*]], [[X:%.*]] +; CHECK-NEXT: [[NAND:%.*]] = xor <2 x i4> [[AND]], +; CHECK-NEXT: ret <2 x i4> [[NAND]] +; + %and = and <2 x i4> %y, %x + %xor = xor <2 x i4> %x, %y + %nand = xor <2 x i4> %and, + %or = or <2 x i4> %xor, %nand + ret <2 x i4> %or +} + declare i32 @llvm.fshl.i32 (i32, i32, i32) declare i32 @llvm.fshr.i32 (i32, i32, i32) diff --git a/llvm/test/Transforms/InstSimplify/ptrmask.ll b/llvm/test/Transforms/InstSimplify/ptrmask.ll index dd83abfdeee4..d2c4a5dd7f03 100644 --- a/llvm/test/Transforms/InstSimplify/ptrmask.ll +++ b/llvm/test/Transforms/InstSimplify/ptrmask.ll @@ -40,7 +40,8 @@ define <2 x ptr addrspace(1) > @ptrmask_simplify_poison_and_zero_i32_vec_fail(<2 define <2 x ptr> @ptrmask_simplify_undef_and_ones_vec(<2 x ptr> %p) { ; CHECK-LABEL: define <2 x ptr> @ptrmask_simplify_undef_and_ones_vec ; CHECK-SAME: (<2 x ptr> [[P:%.*]]) { -; CHECK-NEXT: ret <2 x ptr> [[P]] +; CHECK-NEXT: [[R:%.*]] = call <2 x ptr> @llvm.ptrmask.v2p0.v2i64(<2 x ptr> [[P]], <2 x i64> ) +; CHECK-NEXT: ret <2 x ptr> [[R]] ; %r = call <2 x ptr> @llvm.ptrmask.v2p1.v2i64(<2 x ptr> %p, <2 x i64> ) ret <2 x ptr> %r diff --git a/llvm/test/Transforms/InstSimplify/rem.ll b/llvm/test/Transforms/InstSimplify/rem.ll index 5af3b5f7c5e0..a46db0342042 100644 --- a/llvm/test/Transforms/InstSimplify/rem.ll +++ b/llvm/test/Transforms/InstSimplify/rem.ll @@ -17,11 +17,11 @@ define <2 x i32> @zero_dividend_vector(<2 x i32> %A) { ret <2 x i32> %B } -define <2 x i32> @zero_dividend_vector_undef_elt(<2 x i32> %A) { -; CHECK-LABEL: @zero_dividend_vector_undef_elt( +define <2 x i32> @zero_dividend_vector_poison_elt(<2 x i32> %A) { +; CHECK-LABEL: @zero_dividend_vector_poison_elt( ; CHECK-NEXT: ret <2 x i32> zeroinitializer ; - %B = urem <2 x i32> , %A + %B = urem <2 x i32> , %A ret <2 x i32> %B } diff --git a/llvm/test/Transforms/InstSimplify/saturating-add-sub.ll b/llvm/test/Transforms/InstSimplify/saturating-add-sub.ll index 6fb12612f2f7..40b22c619f76 100644 --- a/llvm/test/Transforms/InstSimplify/saturating-add-sub.ll +++ b/llvm/test/Transforms/InstSimplify/saturating-add-sub.ll @@ -44,7 +44,7 @@ define <2 x i8> @uadd_vector_0_commute(<2 x i8> %a) { ; CHECK-LABEL: @uadd_vector_0_commute( ; CHECK-NEXT: ret <2 x i8> [[A:%.*]] ; - %x2v = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> , <2 x i8> %a) + %x2v = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> , <2 x i8> %a) ret <2 x i8> %x2v } @@ -156,7 +156,7 @@ define <2 x i8> @sadd_vector_0(<2 x i8> %a) { ; CHECK-LABEL: @sadd_vector_0( ; CHECK-NEXT: ret <2 x i8> [[A:%.*]] ; - %y1v = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> %a, <2 x i8> ) + %y1v = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> %a, <2 x i8> ) ret <2 x i8> %y1v } @@ -205,10 +205,10 @@ define i8 @sadd_scalar_maxval_commute(i8 %a) { define <2 x i8> @sadd_vector_maxval_commute(<2 x i8> %a) { ; CHECK-LABEL: @sadd_vector_maxval_commute( -; CHECK-NEXT: [[Y4V:%.*]] = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> , <2 x i8> [[A:%.*]]) +; CHECK-NEXT: [[Y4V:%.*]] = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> , <2 x i8> [[A:%.*]]) ; CHECK-NEXT: ret <2 x i8> [[Y4V]] ; - %y4v = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> , <2 x i8> %a) + %y4v = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> , <2 x i8> %a) ret <2 x i8> %y4v } diff --git a/llvm/test/Transforms/InstSimplify/sdiv.ll b/llvm/test/Transforms/InstSimplify/sdiv.ll index 2514d90b0123..99092802cab0 100644 --- a/llvm/test/Transforms/InstSimplify/sdiv.ll +++ b/llvm/test/Transforms/InstSimplify/sdiv.ll @@ -158,11 +158,11 @@ define <2 x i32> @knownnegation_commute_vec_bad3(<2 x i32> %x, <2 x i32> %y) { ret <2 x i32> %div } -define <3 x i32> @negated_operand_vec_undef(<3 x i32> %x) { -; CHECK-LABEL: @negated_operand_vec_undef( +define <3 x i32> @negated_operand_vec_poison(<3 x i32> %x) { +; CHECK-LABEL: @negated_operand_vec_poison( ; CHECK-NEXT: ret <3 x i32> ; - %negx = sub nsw <3 x i32> , %x + %negx = sub nsw <3 x i32> , %x %div = sdiv <3 x i32> %negx, %x ret <3 x i32> %div } diff --git a/llvm/test/Transforms/InstSimplify/select-inseltpoison.ll b/llvm/test/Transforms/InstSimplify/select-inseltpoison.ll index 2a4ce85ed11f..fcf8c31b25ee 100644 --- a/llvm/test/Transforms/InstSimplify/select-inseltpoison.ll +++ b/llvm/test/Transforms/InstSimplify/select-inseltpoison.ll @@ -17,11 +17,11 @@ define <2 x i1> @bool_true_or_false_vec(<2 x i1> %cond) { ret <2 x i1> %s } -define <2 x i1> @bool_true_or_false_vec_undef(<2 x i1> %cond) { -; CHECK-LABEL: @bool_true_or_false_vec_undef( +define <2 x i1> @bool_true_or_false_vec_poison(<2 x i1> %cond) { +; CHECK-LABEL: @bool_true_or_false_vec_poison( ; CHECK-NEXT: ret <2 x i1> [[COND:%.*]] ; - %s = select <2 x i1> %cond, <2 x i1> , <2 x i1> + %s = select <2 x i1> %cond, <2 x i1> , <2 x i1> ret <2 x i1> %s } @@ -57,27 +57,27 @@ define <2 x i32> @equal_arms_vec(<2 x i1> %cond, <2 x i32> %x) { ret <2 x i32> %V } -define <2 x i32> @equal_arms_vec_undef(<2 x i1> %cond) { -; CHECK-LABEL: @equal_arms_vec_undef( +define <2 x i32> @equal_arms_vec_poison(<2 x i1> %cond) { +; CHECK-LABEL: @equal_arms_vec_poison( ; CHECK-NEXT: ret <2 x i32> ; - %V = select <2 x i1> %cond, <2 x i32> , <2 x i32> + %V = select <2 x i1> %cond, <2 x i32> , <2 x i32> ret <2 x i32> %V } -define <3 x float> @equal_arms_vec_less_undef(<3 x i1> %cond) { -; CHECK-LABEL: @equal_arms_vec_less_undef( +define <3 x float> @equal_arms_vec_less_poison(<3 x i1> %cond) { +; CHECK-LABEL: @equal_arms_vec_less_poison( ; CHECK-NEXT: ret <3 x float> ; - %V = select <3 x i1> %cond, <3 x float> , <3 x float> + %V = select <3 x i1> %cond, <3 x float> , <3 x float> ret <3 x float> %V } -define <3 x float> @equal_arms_vec_more_undef(<3 x i1> %cond) { -; CHECK-LABEL: @equal_arms_vec_more_undef( -; CHECK-NEXT: ret <3 x float> +define <3 x float> @equal_arms_vec_more_poison(<3 x i1> %cond) { +; CHECK-LABEL: @equal_arms_vec_more_poison( +; CHECK-NEXT: ret <3 x float> ; - %V = select <3 x i1> %cond, <3 x float> , <3 x float> + %V = select <3 x i1> %cond, <3 x float> , <3 x float> ret <3 x float> %V } @@ -105,19 +105,19 @@ define <2 x i8> @vsel_mixedvec() { ret <2 x i8> %s } -define <3 x i8> @vsel_undef_true_op(<3 x i8> %x, <3 x i8> %y) { -; CHECK-LABEL: @vsel_undef_true_op( +define <3 x i8> @vsel_poison_true_op(<3 x i8> %x, <3 x i8> %y) { +; CHECK-LABEL: @vsel_poison_true_op( ; CHECK-NEXT: ret <3 x i8> [[X:%.*]] ; - %s = select <3 x i1>, <3 x i8> %x, <3 x i8> %y + %s = select <3 x i1>, <3 x i8> %x, <3 x i8> %y ret <3 x i8> %s } -define <3 x i4> @vsel_undef_false_op(<3 x i4> %x, <3 x i4> %y) { -; CHECK-LABEL: @vsel_undef_false_op( +define <3 x i4> @vsel_poison_false_op(<3 x i4> %x, <3 x i4> %y) { +; CHECK-LABEL: @vsel_poison_false_op( ; CHECK-NEXT: ret <3 x i4> [[Y:%.*]] ; - %s = select <3 x i1>, <3 x i4> %x, <3 x i4> %y + %s = select <3 x i1>, <3 x i4> %x, <3 x i4> %y ret <3 x i4> %s } diff --git a/llvm/test/Transforms/InstSimplify/select.ll b/llvm/test/Transforms/InstSimplify/select.ll index fe93a0c3f212..40c1460e3ebc 100644 --- a/llvm/test/Transforms/InstSimplify/select.ll +++ b/llvm/test/Transforms/InstSimplify/select.ll @@ -25,11 +25,11 @@ define <2 x i1> @bool_true_or_false_vec(<2 x i1> %cond) { ret <2 x i1> %s } -define <2 x i1> @bool_true_or_false_vec_undef(<2 x i1> %cond) { -; CHECK-LABEL: @bool_true_or_false_vec_undef( +define <2 x i1> @bool_true_or_false_vec_poison(<2 x i1> %cond) { +; CHECK-LABEL: @bool_true_or_false_vec_poison( ; CHECK-NEXT: ret <2 x i1> [[COND:%.*]] ; - %s = select <2 x i1> %cond, <2 x i1> , <2 x i1> + %s = select <2 x i1> %cond, <2 x i1> , <2 x i1> ret <2 x i1> %s } @@ -65,27 +65,27 @@ define <2 x i32> @equal_arms_vec(<2 x i1> %cond, <2 x i32> %x) { ret <2 x i32> %V } -define <2 x i32> @equal_arms_vec_undef(<2 x i1> %cond) { -; CHECK-LABEL: @equal_arms_vec_undef( +define <2 x i32> @equal_arms_vec_poison(<2 x i1> %cond) { +; CHECK-LABEL: @equal_arms_vec_poison( ; CHECK-NEXT: ret <2 x i32> ; - %V = select <2 x i1> %cond, <2 x i32> , <2 x i32> + %V = select <2 x i1> %cond, <2 x i32> , <2 x i32> ret <2 x i32> %V } -define <3 x float> @equal_arms_vec_less_undef(<3 x i1> %cond) { -; CHECK-LABEL: @equal_arms_vec_less_undef( +define <3 x float> @equal_arms_vec_less_poison(<3 x i1> %cond) { +; CHECK-LABEL: @equal_arms_vec_less_poison( ; CHECK-NEXT: ret <3 x float> ; - %V = select <3 x i1> %cond, <3 x float> , <3 x float> + %V = select <3 x i1> %cond, <3 x float> , <3 x float> ret <3 x float> %V } -define <3 x float> @equal_arms_vec_more_undef(<3 x i1> %cond) { -; CHECK-LABEL: @equal_arms_vec_more_undef( -; CHECK-NEXT: ret <3 x float> +define <3 x float> @equal_arms_vec_more_poison(<3 x i1> %cond) { +; CHECK-LABEL: @equal_arms_vec_more_poison( +; CHECK-NEXT: ret <3 x float> ; - %V = select <3 x i1> %cond, <3 x float> , <3 x float> + %V = select <3 x i1> %cond, <3 x float> , <3 x float> ret <3 x float> %V } @@ -113,19 +113,19 @@ define <2 x i8> @vsel_mixedvec() { ret <2 x i8> %s } -define <3 x i8> @vsel_undef_true_op(<3 x i8> %x, <3 x i8> %y) { -; CHECK-LABEL: @vsel_undef_true_op( +define <3 x i8> @vsel_poison_true_op(<3 x i8> %x, <3 x i8> %y) { +; CHECK-LABEL: @vsel_poison_true_op( ; CHECK-NEXT: ret <3 x i8> [[X:%.*]] ; - %s = select <3 x i1>, <3 x i8> %x, <3 x i8> %y + %s = select <3 x i1>, <3 x i8> %x, <3 x i8> %y ret <3 x i8> %s } -define <3 x i4> @vsel_undef_false_op(<3 x i4> %x, <3 x i4> %y) { -; CHECK-LABEL: @vsel_undef_false_op( +define <3 x i4> @vsel_poison_false_op(<3 x i4> %x, <3 x i4> %y) { +; CHECK-LABEL: @vsel_poison_false_op( ; CHECK-NEXT: ret <3 x i4> [[Y:%.*]] ; - %s = select <3 x i1>, <3 x i4> %x, <3 x i4> %y + %s = select <3 x i1>, <3 x i4> %x, <3 x i4> %y ret <3 x i4> %s } diff --git a/llvm/test/Transforms/InstSimplify/shift.ll b/llvm/test/Transforms/InstSimplify/shift.ll index b562c3c164d5..a816fcbdeeee 100644 --- a/llvm/test/Transforms/InstSimplify/shift.ll +++ b/llvm/test/Transforms/InstSimplify/shift.ll @@ -17,11 +17,11 @@ define i41 @shl_0(i41 %X) { ret i41 %B } -define <2 x i41> @shl_0_vec_undef_elt(<2 x i41> %X) { -; CHECK-LABEL: @shl_0_vec_undef_elt( +define <2 x i41> @shl_0_vec_poison_elt(<2 x i41> %X) { +; CHECK-LABEL: @shl_0_vec_poison_elt( ; CHECK-NEXT: ret <2 x i41> zeroinitializer ; - %B = shl <2 x i41> , %X + %B = shl <2 x i41> , %X ret <2 x i41> %B } @@ -41,11 +41,11 @@ define i39 @ashr_0(i39 %X) { ret i39 %B } -define <2 x i141> @ashr_0_vec_undef_elt(<2 x i141> %X) { -; CHECK-LABEL: @ashr_0_vec_undef_elt( +define <2 x i141> @ashr_0_vec_poison_elt(<2 x i141> %X) { +; CHECK-LABEL: @ashr_0_vec_poison_elt( ; CHECK-NEXT: ret <2 x i141> zeroinitializer ; - %B = shl <2 x i141> , %X + %B = shl <2 x i141> , %X ret <2 x i141> %B } @@ -113,11 +113,11 @@ define i32 @ashr_all_ones(i32 %A) { ret i32 %B } -define <3 x i8> @ashr_all_ones_vec_with_undef_elts(<3 x i8> %x, <3 x i8> %y) { -; CHECK-LABEL: @ashr_all_ones_vec_with_undef_elts( +define <3 x i8> @ashr_all_ones_vec_with_poison_elts(<3 x i8> %x, <3 x i8> %y) { +; CHECK-LABEL: @ashr_all_ones_vec_with_poison_elts( ; CHECK-NEXT: ret <3 x i8> ; - %sh = ashr <3 x i8> , %y + %sh = ashr <3 x i8> , %y ret <3 x i8> %sh } @@ -306,11 +306,22 @@ define <2 x i7> @all_ones_left_right_splat(<2 x i7> %x) { ; Poison could propagate, but undef must not. -define <3 x i7> @all_ones_left_right_splat_poison_undef_elt(<3 x i7> %x) { -; CHECK-LABEL: @all_ones_left_right_splat_poison_undef_elt( +define <3 x i7> @all_ones_left_right_splat_undef_elt(<3 x i7> %x) { +; CHECK-LABEL: @all_ones_left_right_splat_undef_elt( +; CHECK-NEXT: [[LEFT:%.*]] = shl <3 x i7> , [[X:%.*]] +; CHECK-NEXT: [[RIGHT:%.*]] = ashr <3 x i7> [[LEFT]], [[X]] +; CHECK-NEXT: ret <3 x i7> [[RIGHT]] +; + %left = shl <3 x i7> , %x + %right = ashr <3 x i7> %left, %x + ret <3 x i7> %right +} + +define <3 x i7> @all_ones_left_right_splat_poison__elt(<3 x i7> %x) { +; CHECK-LABEL: @all_ones_left_right_splat_poison__elt( ; CHECK-NEXT: ret <3 x i7> ; - %left = shl <3 x i7> , %x + %left = shl <3 x i7> , %x %right = ashr <3 x i7> %left, %x ret <3 x i7> %right } diff --git a/llvm/test/Transforms/InstSimplify/srem.ll b/llvm/test/Transforms/InstSimplify/srem.ll index b1cbdf35b3c7..ab726832e517 100644 --- a/llvm/test/Transforms/InstSimplify/srem.ll +++ b/llvm/test/Transforms/InstSimplify/srem.ll @@ -39,11 +39,11 @@ define <2 x i32> @knownnegation_commute_vec(<2 x i32> %x, <2 x i32> %y) { ret <2 x i32> %rem } -define <3 x i32> @negated_operand_vec_undef(<3 x i32> %x) { -; CHECK-LABEL: @negated_operand_vec_undef( +define <3 x i32> @negated_operand_vec_poison(<3 x i32> %x) { +; CHECK-LABEL: @negated_operand_vec_poison( ; CHECK-NEXT: ret <3 x i32> zeroinitializer ; - %negx = sub <3 x i32> , %x + %negx = sub <3 x i32> , %x %rem = srem <3 x i32> %negx, %x ret <3 x i32> %rem } diff --git a/llvm/test/Transforms/InstSimplify/sub.ll b/llvm/test/Transforms/InstSimplify/sub.ll index deb0ee33cd92..fd88fc15716c 100644 --- a/llvm/test/Transforms/InstSimplify/sub.ll +++ b/llvm/test/Transforms/InstSimplify/sub.ll @@ -29,7 +29,7 @@ define <2 x i32> @sub_zero_vec(<2 x i32> %A) { ; CHECK-LABEL: @sub_zero_vec( ; CHECK-NEXT: ret <2 x i32> [[A:%.*]] ; - %B = sub <2 x i32> %A, + %B = sub <2 x i32> %A, ret <2 x i32> %B } @@ -46,8 +46,8 @@ define <2 x i32> @neg_neg_vec(<2 x i32> %A) { ; CHECK-LABEL: @neg_neg_vec( ; CHECK-NEXT: ret <2 x i32> [[A:%.*]] ; - %B = sub <2 x i32> , %A - %C = sub <2 x i32> , %B + %B = sub <2 x i32> , %A + %C = sub <2 x i32> , %B ret <2 x i32> %C } diff --git a/llvm/test/Transforms/InstSimplify/xor.ll b/llvm/test/Transforms/InstSimplify/xor.ll index 0e23cc66c165..229e943a3836 100644 --- a/llvm/test/Transforms/InstSimplify/xor.ll +++ b/llvm/test/Transforms/InstSimplify/xor.ll @@ -156,6 +156,20 @@ define <2 x i4> @xor_and_or_not_undef_elt(<2 x i4> %a, <2 x i4> %b) { ret <2 x i4> %r } +; but correct to propagate poison element + +define <2 x i4> @xor_and_or_not_poison_elt(<2 x i4> %a, <2 x i4> %b) { +; CHECK-LABEL: @xor_and_or_not_poison_elt( +; CHECK-NEXT: [[NOT:%.*]] = xor <2 x i4> [[A:%.*]], +; CHECK-NEXT: ret <2 x i4> [[NOT]] +; + %and = and <2 x i4> %b, %a + %not = xor <2 x i4> %a, + %or = or <2 x i4> %not, %b + %r = xor <2 x i4> %or, %and + ret <2 x i4> %r +} + define i4 @xor_or_and_not_commute0(i4 %a, i4 %b) { ; CHECK-LABEL: @xor_or_and_not_commute0( ; CHECK-NEXT: ret i4 [[A:%.*]] @@ -277,11 +291,11 @@ define i4 @xor_or_and_not_wrong_val2(i4 %a, i4 %b, i4 %c) { ret i4 %r } -define <2 x i4> @xor_or_and_not_undef_elt(<2 x i4> %a, <2 x i4> %b) { -; CHECK-LABEL: @xor_or_and_not_undef_elt( +define <2 x i4> @xor_or_and_not_poison_elt(<2 x i4> %a, <2 x i4> %b) { +; CHECK-LABEL: @xor_or_and_not_poison_elt( ; CHECK-NEXT: ret <2 x i4> [[A:%.*]] ; - %not = xor <2 x i4> %a, + %not = xor <2 x i4> %a, %and = and <2 x i4> %b, %not %or = or <2 x i4> %a, %b %r = xor <2 x i4> %or, %and diff --git a/llvm/test/Transforms/Reassociate/inverses.ll b/llvm/test/Transforms/Reassociate/inverses.ll index b6962c6872a9..a9d0c4fb0322 100644 --- a/llvm/test/Transforms/Reassociate/inverses.ll +++ b/llvm/test/Transforms/Reassociate/inverses.ll @@ -12,12 +12,12 @@ define i32 @test1(i32 %a, i32 %b) { ret i32 %t5 } -define <2 x i32> @not_op_vec_undef(<2 x i32> %a, <2 x i32> %b) { -; CHECK-LABEL: @not_op_vec_undef( +define <2 x i32> @not_op_vec_poison(<2 x i32> %a, <2 x i32> %b) { +; CHECK-LABEL: @not_op_vec_poison( ; CHECK-NEXT: ret <2 x i32> zeroinitializer ; %t2 = and <2 x i32> %b, %a - %t4 = xor <2 x i32> %a, + %t4 = xor <2 x i32> %a, %t5 = and <2 x i32> %t2, %t4 ret <2 x i32> %t5 } diff --git a/llvm/test/Transforms/Reassociate/negation.ll b/llvm/test/Transforms/Reassociate/negation.ll index 4718d9d87ae1..14ae86fb94aa 100644 --- a/llvm/test/Transforms/Reassociate/negation.ll +++ b/llvm/test/Transforms/Reassociate/negation.ll @@ -31,16 +31,16 @@ define i32 @test2(i32 %a, i32 %b, i32 %z) { ret i32 %f } -define <2 x i32> @negate_vec_undefs(<2 x i32> %a, <2 x i32> %b, <2 x i32> %z) { -; CHECK-LABEL: @negate_vec_undefs( +define <2 x i32> @negate_vec_poisons(<2 x i32> %a, <2 x i32> %b, <2 x i32> %z) { +; CHECK-LABEL: @negate_vec_poisons( ; CHECK-NEXT: [[E:%.*]] = mul <2 x i32> [[A:%.*]], ; CHECK-NEXT: [[F:%.*]] = mul <2 x i32> [[E]], [[Z:%.*]] ; CHECK-NEXT: ret <2 x i32> [[F]] ; %d = mul <2 x i32> %z, - %c = sub <2 x i32> , %d + %c = sub <2 x i32> , %d %e = mul <2 x i32> %a, %c - %f = sub <2 x i32> , %e + %f = sub <2 x i32> , %e ret <2 x i32> %f } diff --git a/llvm/unittests/IR/ConstantsTest.cpp b/llvm/unittests/IR/ConstantsTest.cpp index 1d6a92c498b0..8f0a507c0fd1 100644 --- a/llvm/unittests/IR/ConstantsTest.cpp +++ b/llvm/unittests/IR/ConstantsTest.cpp @@ -581,7 +581,7 @@ TEST(ConstantsTest, containsUndefElemTest) { } } -// Check that undefined elements in vector constants are matched +// Check that poison elements in vector constants are matched // correctly for both integer and floating-point types. Just don't // crash on vectors of pointers (could be handled?). @@ -590,6 +590,7 @@ TEST(ConstantsTest, isElementWiseEqual) { Type *Int32Ty = Type::getInt32Ty(Context); Constant *CU = UndefValue::get(Int32Ty); + Constant *CP = PoisonValue::get(Int32Ty); Constant *C1 = ConstantInt::get(Int32Ty, 1); Constant *C2 = ConstantInt::get(Int32Ty, 2); @@ -597,15 +598,25 @@ TEST(ConstantsTest, isElementWiseEqual) { Constant *C12U1 = ConstantVector::get({C1, C2, CU, C1}); Constant *C12U2 = ConstantVector::get({C1, C2, CU, C2}); Constant *C12U21 = ConstantVector::get({C1, C2, CU, C2, C1}); + Constant *C12P1 = ConstantVector::get({C1, C2, CP, C1}); + Constant *C12P2 = ConstantVector::get({C1, C2, CP, C2}); + Constant *C12P21 = ConstantVector::get({C1, C2, CP, C2, C1}); - EXPECT_TRUE(C1211->isElementWiseEqual(C12U1)); - EXPECT_TRUE(C12U1->isElementWiseEqual(C1211)); + EXPECT_FALSE(C1211->isElementWiseEqual(C12U1)); + EXPECT_FALSE(C12U1->isElementWiseEqual(C1211)); EXPECT_FALSE(C12U2->isElementWiseEqual(C12U1)); EXPECT_FALSE(C12U1->isElementWiseEqual(C12U2)); EXPECT_FALSE(C12U21->isElementWiseEqual(C12U2)); + EXPECT_TRUE(C1211->isElementWiseEqual(C12P1)); + EXPECT_TRUE(C12P1->isElementWiseEqual(C1211)); + EXPECT_FALSE(C12P2->isElementWiseEqual(C12P1)); + EXPECT_FALSE(C12P1->isElementWiseEqual(C12P2)); + EXPECT_FALSE(C12P21->isElementWiseEqual(C12P2)); + Type *FltTy = Type::getFloatTy(Context); Constant *CFU = UndefValue::get(FltTy); + Constant *CFP = PoisonValue::get(FltTy); Constant *CF1 = ConstantFP::get(FltTy, 1.0); Constant *CF2 = ConstantFP::get(FltTy, 2.0); @@ -613,25 +624,41 @@ TEST(ConstantsTest, isElementWiseEqual) { Constant *CF12U1 = ConstantVector::get({CF1, CF2, CFU, CF1}); Constant *CF12U2 = ConstantVector::get({CF1, CF2, CFU, CF2}); Constant *CFUU1U = ConstantVector::get({CFU, CFU, CF1, CFU}); + Constant *CF12P1 = ConstantVector::get({CF1, CF2, CFP, CF1}); + Constant *CF12P2 = ConstantVector::get({CF1, CF2, CFP, CF2}); + Constant *CFPP1P = ConstantVector::get({CFP, CFP, CF1, CFP}); - EXPECT_TRUE(CF1211->isElementWiseEqual(CF12U1)); - EXPECT_TRUE(CF12U1->isElementWiseEqual(CF1211)); - EXPECT_TRUE(CFUU1U->isElementWiseEqual(CF12U1)); + EXPECT_FALSE(CF1211->isElementWiseEqual(CF12U1)); + EXPECT_FALSE(CF12U1->isElementWiseEqual(CF1211)); + EXPECT_FALSE(CFUU1U->isElementWiseEqual(CF12U1)); EXPECT_FALSE(CF12U2->isElementWiseEqual(CF12U1)); EXPECT_FALSE(CF12U1->isElementWiseEqual(CF12U2)); + EXPECT_TRUE(CF1211->isElementWiseEqual(CF12P1)); + EXPECT_TRUE(CF12P1->isElementWiseEqual(CF1211)); + EXPECT_TRUE(CFPP1P->isElementWiseEqual(CF12P1)); + EXPECT_FALSE(CF12P2->isElementWiseEqual(CF12P1)); + EXPECT_FALSE(CF12P1->isElementWiseEqual(CF12P2)); + PointerType *PtrTy = PointerType::get(Context, 0); Constant *CPU = UndefValue::get(PtrTy); + Constant *CPP = PoisonValue::get(PtrTy); Constant *CP0 = ConstantPointerNull::get(PtrTy); Constant *CP0000 = ConstantVector::get({CP0, CP0, CP0, CP0}); Constant *CP00U0 = ConstantVector::get({CP0, CP0, CPU, CP0}); Constant *CP00U = ConstantVector::get({CP0, CP0, CPU}); + Constant *CP00P0 = ConstantVector::get({CP0, CP0, CPP, CP0}); + Constant *CP00P = ConstantVector::get({CP0, CP0, CPP}); EXPECT_FALSE(CP0000->isElementWiseEqual(CP00U0)); EXPECT_FALSE(CP00U0->isElementWiseEqual(CP0000)); EXPECT_FALSE(CP0000->isElementWiseEqual(CP00U)); EXPECT_FALSE(CP00U->isElementWiseEqual(CP00U0)); + EXPECT_FALSE(CP0000->isElementWiseEqual(CP00P0)); + EXPECT_FALSE(CP00P0->isElementWiseEqual(CP0000)); + EXPECT_FALSE(CP0000->isElementWiseEqual(CP00P)); + EXPECT_FALSE(CP00P->isElementWiseEqual(CP00P0)); } // Check that vector/aggregate constants correctly store undef and poison diff --git a/llvm/unittests/IR/PatternMatch.cpp b/llvm/unittests/IR/PatternMatch.cpp index 4d0c2e4220fe..133012684d16 100644 --- a/llvm/unittests/IR/PatternMatch.cpp +++ b/llvm/unittests/IR/PatternMatch.cpp @@ -1184,6 +1184,8 @@ TEST_F(PatternMatchTest, VectorUndefInt) { Type *VectorTy = FixedVectorType::get(ScalarTy, 4); Constant *ScalarUndef = UndefValue::get(ScalarTy); Constant *VectorUndef = UndefValue::get(VectorTy); + Constant *ScalarPoison = PoisonValue::get(ScalarTy); + Constant *VectorPoison = PoisonValue::get(VectorTy); Constant *ScalarZero = Constant::getNullValue(ScalarTy); Constant *VectorZero = Constant::getNullValue(VectorTy); @@ -1194,17 +1196,30 @@ TEST_F(PatternMatchTest, VectorUndefInt) { Elems.push_back(ScalarZero); Constant *VectorZeroUndef = ConstantVector::get(Elems); + SmallVector Elems2; + Elems2.push_back(ScalarPoison); + Elems2.push_back(ScalarZero); + Elems2.push_back(ScalarPoison); + Elems2.push_back(ScalarZero); + Constant *VectorZeroPoison = ConstantVector::get(Elems2); + EXPECT_TRUE(match(ScalarUndef, m_Undef())); + EXPECT_TRUE(match(ScalarPoison, m_Undef())); EXPECT_TRUE(match(VectorUndef, m_Undef())); + EXPECT_TRUE(match(VectorPoison, m_Undef())); EXPECT_FALSE(match(ScalarZero, m_Undef())); EXPECT_FALSE(match(VectorZero, m_Undef())); EXPECT_FALSE(match(VectorZeroUndef, m_Undef())); + EXPECT_FALSE(match(VectorZeroPoison, m_Undef())); EXPECT_FALSE(match(ScalarUndef, m_Zero())); + EXPECT_FALSE(match(ScalarPoison, m_Zero())); EXPECT_FALSE(match(VectorUndef, m_Zero())); + EXPECT_FALSE(match(VectorPoison, m_Zero())); + EXPECT_FALSE(match(VectorZeroUndef, m_Zero())); EXPECT_TRUE(match(ScalarZero, m_Zero())); EXPECT_TRUE(match(VectorZero, m_Zero())); - EXPECT_TRUE(match(VectorZeroUndef, m_Zero())); + EXPECT_TRUE(match(VectorZeroPoison, m_Zero())); const APInt *C; // Regardless of whether undefs are allowed, @@ -1249,6 +1264,8 @@ TEST_F(PatternMatchTest, VectorUndefFloat) { Type *VectorTy = FixedVectorType::get(ScalarTy, 4); Constant *ScalarUndef = UndefValue::get(ScalarTy); Constant *VectorUndef = UndefValue::get(VectorTy); + Constant *ScalarPoison = PoisonValue::get(ScalarTy); + Constant *VectorPoison = PoisonValue::get(VectorTy); Constant *ScalarZero = Constant::getNullValue(ScalarTy); Constant *VectorZero = Constant::getNullValue(VectorTy); Constant *ScalarPosInf = ConstantFP::getInfinity(ScalarTy, false); @@ -1258,72 +1275,116 @@ TEST_F(PatternMatchTest, VectorUndefFloat) { Constant *VectorZeroUndef = ConstantVector::get({ScalarUndef, ScalarZero, ScalarUndef, ScalarZero}); + Constant *VectorZeroPoison = + ConstantVector::get({ScalarPoison, ScalarZero, ScalarPoison, ScalarZero}); + Constant *VectorInfUndef = ConstantVector::get( {ScalarPosInf, ScalarNegInf, ScalarUndef, ScalarPosInf}); + Constant *VectorInfPoison = ConstantVector::get( + {ScalarPosInf, ScalarNegInf, ScalarPoison, ScalarPosInf}); + Constant *VectorNaNUndef = ConstantVector::get({ScalarUndef, ScalarNaN, ScalarNaN, ScalarNaN}); + Constant *VectorNaNPoison = + ConstantVector::get({ScalarPoison, ScalarNaN, ScalarNaN, ScalarNaN}); + EXPECT_TRUE(match(ScalarUndef, m_Undef())); EXPECT_TRUE(match(VectorUndef, m_Undef())); + EXPECT_TRUE(match(ScalarPoison, m_Undef())); + EXPECT_TRUE(match(VectorPoison, m_Undef())); EXPECT_FALSE(match(ScalarZero, m_Undef())); EXPECT_FALSE(match(VectorZero, m_Undef())); EXPECT_FALSE(match(VectorZeroUndef, m_Undef())); EXPECT_FALSE(match(VectorInfUndef, m_Undef())); EXPECT_FALSE(match(VectorNaNUndef, m_Undef())); + EXPECT_FALSE(match(VectorZeroPoison, m_Undef())); + EXPECT_FALSE(match(VectorInfPoison, m_Undef())); + EXPECT_FALSE(match(VectorNaNPoison, m_Undef())); EXPECT_FALSE(match(ScalarUndef, m_AnyZeroFP())); EXPECT_FALSE(match(VectorUndef, m_AnyZeroFP())); + EXPECT_FALSE(match(ScalarPoison, m_AnyZeroFP())); + EXPECT_FALSE(match(VectorPoison, m_AnyZeroFP())); EXPECT_TRUE(match(ScalarZero, m_AnyZeroFP())); EXPECT_TRUE(match(VectorZero, m_AnyZeroFP())); - EXPECT_TRUE(match(VectorZeroUndef, m_AnyZeroFP())); + EXPECT_FALSE(match(VectorZeroUndef, m_AnyZeroFP())); EXPECT_FALSE(match(VectorInfUndef, m_AnyZeroFP())); EXPECT_FALSE(match(VectorNaNUndef, m_AnyZeroFP())); + EXPECT_TRUE(match(VectorZeroPoison, m_AnyZeroFP())); + EXPECT_FALSE(match(VectorInfPoison, m_AnyZeroFP())); + EXPECT_FALSE(match(VectorNaNPoison, m_AnyZeroFP())); EXPECT_FALSE(match(ScalarUndef, m_NaN())); EXPECT_FALSE(match(VectorUndef, m_NaN())); EXPECT_FALSE(match(VectorZeroUndef, m_NaN())); + EXPECT_FALSE(match(ScalarPoison, m_NaN())); + EXPECT_FALSE(match(VectorPoison, m_NaN())); + EXPECT_FALSE(match(VectorZeroPoison, m_NaN())); EXPECT_FALSE(match(ScalarPosInf, m_NaN())); EXPECT_FALSE(match(ScalarNegInf, m_NaN())); EXPECT_TRUE(match(ScalarNaN, m_NaN())); EXPECT_FALSE(match(VectorInfUndef, m_NaN())); - EXPECT_TRUE(match(VectorNaNUndef, m_NaN())); + EXPECT_FALSE(match(VectorNaNUndef, m_NaN())); + EXPECT_FALSE(match(VectorInfPoison, m_NaN())); + EXPECT_TRUE(match(VectorNaNPoison, m_NaN())); EXPECT_FALSE(match(ScalarUndef, m_NonNaN())); EXPECT_FALSE(match(VectorUndef, m_NonNaN())); - EXPECT_TRUE(match(VectorZeroUndef, m_NonNaN())); + EXPECT_FALSE(match(VectorZeroUndef, m_NonNaN())); + EXPECT_FALSE(match(ScalarPoison, m_NonNaN())); + EXPECT_FALSE(match(VectorPoison, m_NonNaN())); + EXPECT_TRUE(match(VectorZeroPoison, m_NonNaN())); EXPECT_TRUE(match(ScalarPosInf, m_NonNaN())); EXPECT_TRUE(match(ScalarNegInf, m_NonNaN())); EXPECT_FALSE(match(ScalarNaN, m_NonNaN())); - EXPECT_TRUE(match(VectorInfUndef, m_NonNaN())); + EXPECT_FALSE(match(VectorInfUndef, m_NonNaN())); EXPECT_FALSE(match(VectorNaNUndef, m_NonNaN())); + EXPECT_TRUE(match(VectorInfPoison, m_NonNaN())); + EXPECT_FALSE(match(VectorNaNPoison, m_NonNaN())); EXPECT_FALSE(match(ScalarUndef, m_Inf())); EXPECT_FALSE(match(VectorUndef, m_Inf())); EXPECT_FALSE(match(VectorZeroUndef, m_Inf())); + EXPECT_FALSE(match(ScalarPoison, m_Inf())); + EXPECT_FALSE(match(VectorPoison, m_Inf())); + EXPECT_FALSE(match(VectorZeroPoison, m_Inf())); EXPECT_TRUE(match(ScalarPosInf, m_Inf())); EXPECT_TRUE(match(ScalarNegInf, m_Inf())); EXPECT_FALSE(match(ScalarNaN, m_Inf())); - EXPECT_TRUE(match(VectorInfUndef, m_Inf())); + EXPECT_FALSE(match(VectorInfUndef, m_Inf())); EXPECT_FALSE(match(VectorNaNUndef, m_Inf())); + EXPECT_TRUE(match(VectorInfPoison, m_Inf())); + EXPECT_FALSE(match(VectorNaNPoison, m_Inf())); EXPECT_FALSE(match(ScalarUndef, m_NonInf())); EXPECT_FALSE(match(VectorUndef, m_NonInf())); - EXPECT_TRUE(match(VectorZeroUndef, m_NonInf())); + EXPECT_FALSE(match(VectorZeroUndef, m_NonInf())); + EXPECT_FALSE(match(ScalarPoison, m_NonInf())); + EXPECT_FALSE(match(VectorPoison, m_NonInf())); + EXPECT_TRUE(match(VectorZeroPoison, m_NonInf())); EXPECT_FALSE(match(ScalarPosInf, m_NonInf())); EXPECT_FALSE(match(ScalarNegInf, m_NonInf())); EXPECT_TRUE(match(ScalarNaN, m_NonInf())); EXPECT_FALSE(match(VectorInfUndef, m_NonInf())); - EXPECT_TRUE(match(VectorNaNUndef, m_NonInf())); + EXPECT_FALSE(match(VectorNaNUndef, m_NonInf())); + EXPECT_FALSE(match(VectorInfPoison, m_NonInf())); + EXPECT_TRUE(match(VectorNaNPoison, m_NonInf())); EXPECT_FALSE(match(ScalarUndef, m_Finite())); EXPECT_FALSE(match(VectorUndef, m_Finite())); - EXPECT_TRUE(match(VectorZeroUndef, m_Finite())); + EXPECT_FALSE(match(VectorZeroUndef, m_Finite())); + EXPECT_FALSE(match(ScalarPoison, m_Finite())); + EXPECT_FALSE(match(VectorPoison, m_Finite())); + EXPECT_TRUE(match(VectorZeroPoison, m_Finite())); EXPECT_FALSE(match(ScalarPosInf, m_Finite())); EXPECT_FALSE(match(ScalarNegInf, m_Finite())); EXPECT_FALSE(match(ScalarNaN, m_Finite())); EXPECT_FALSE(match(VectorInfUndef, m_Finite())); EXPECT_FALSE(match(VectorNaNUndef, m_Finite())); + EXPECT_FALSE(match(VectorInfPoison, m_Finite())); + EXPECT_FALSE(match(VectorNaNPoison, m_Finite())); const APFloat *C; // Regardless of whether undefs are allowed, @@ -1707,38 +1768,57 @@ TEST_F(PatternMatchTest, ConstantPredicateType) { Constant *CMixedU32 = ConstantVector::get({CU32Max, CU32Zero, CU32DeadBeef}); Constant *CU32Undef = UndefValue::get(U32Ty); + Constant *CU32Poison = PoisonValue::get(U32Ty); Constant *CU32MaxWithUndef = ConstantVector::get({CU32Undef, CU32Max, CU32Undef}); + Constant *CU32MaxWithPoison = + ConstantVector::get({CU32Poison, CU32Max, CU32Poison}); EXPECT_FALSE(match(CMixedU32, cst_pred_ty())); EXPECT_FALSE(match(CMixedU32, cst_pred_ty())); EXPECT_TRUE(match(CMixedU32, cst_pred_ty>())); EXPECT_FALSE(match(CMixedU32, cst_pred_ty>())); - EXPECT_TRUE(match(CU32MaxWithUndef, cst_pred_ty())); + EXPECT_FALSE(match(CU32MaxWithUndef, cst_pred_ty())); EXPECT_FALSE(match(CU32MaxWithUndef, cst_pred_ty())); - EXPECT_TRUE(match(CU32MaxWithUndef, cst_pred_ty>())); + EXPECT_FALSE(match(CU32MaxWithUndef, cst_pred_ty>())); EXPECT_FALSE( match(CU32MaxWithUndef, cst_pred_ty>())); + EXPECT_TRUE(match(CU32MaxWithPoison, cst_pred_ty())); + EXPECT_FALSE(match(CU32MaxWithPoison, cst_pred_ty())); + EXPECT_TRUE(match(CU32MaxWithPoison, cst_pred_ty>())); + EXPECT_FALSE( + match(CU32MaxWithPoison, cst_pred_ty>())); + // Float arbitrary vector Constant *CMixedF32 = ConstantVector::get({CF32NaN, CF32Zero, CF32Pi}); Constant *CF32Undef = UndefValue::get(F32Ty); + Constant *CF32Poison = PoisonValue::get(F32Ty); Constant *CF32NaNWithUndef = ConstantVector::get({CF32Undef, CF32NaN, CF32Undef}); + Constant *CF32NaNWithPoison = + ConstantVector::get({CF32Poison, CF32NaN, CF32Poison}); EXPECT_FALSE(match(CMixedF32, cstfp_pred_ty())); EXPECT_FALSE(match(CMixedF32, cstfp_pred_ty())); EXPECT_TRUE(match(CMixedF32, cstfp_pred_ty>())); EXPECT_FALSE(match(CMixedF32, cstfp_pred_ty>())); - EXPECT_TRUE(match(CF32NaNWithUndef, cstfp_pred_ty())); + EXPECT_FALSE(match(CF32NaNWithUndef, cstfp_pred_ty())); EXPECT_FALSE(match(CF32NaNWithUndef, cstfp_pred_ty())); - EXPECT_TRUE( + EXPECT_FALSE( match(CF32NaNWithUndef, cstfp_pred_ty>())); EXPECT_FALSE( match(CF32NaNWithUndef, cstfp_pred_ty>())); + + EXPECT_TRUE(match(CF32NaNWithPoison, cstfp_pred_ty())); + EXPECT_FALSE(match(CF32NaNWithPoison, cstfp_pred_ty())); + EXPECT_TRUE( + match(CF32NaNWithPoison, cstfp_pred_ty>())); + EXPECT_FALSE( + match(CF32NaNWithPoison, cstfp_pred_ty>())); } TEST_F(PatternMatchTest, InsertValue) { @@ -1888,35 +1968,44 @@ TEST_F(PatternMatchTest, NotForbidUndef) { Type *ScalarTy = IRB.getInt8Ty(); Type *VectorTy = FixedVectorType::get(ScalarTy, 3); Constant *ScalarUndef = UndefValue::get(ScalarTy); + Constant *ScalarPoison = PoisonValue::get(ScalarTy); Constant *ScalarOnes = Constant::getAllOnesValue(ScalarTy); Constant *VectorZero = Constant::getNullValue(VectorTy); Constant *VectorOnes = Constant::getAllOnesValue(VectorTy); - SmallVector MixedElems; - MixedElems.push_back(ScalarOnes); - MixedElems.push_back(ScalarOnes); - MixedElems.push_back(ScalarUndef); - Constant *VectorMixed = ConstantVector::get(MixedElems); + SmallVector MixedElemsUndef; + MixedElemsUndef.push_back(ScalarOnes); + MixedElemsUndef.push_back(ScalarOnes); + MixedElemsUndef.push_back(ScalarUndef); + Constant *VectorMixedUndef = ConstantVector::get(MixedElemsUndef); + + SmallVector MixedElemsPoison; + MixedElemsPoison.push_back(ScalarOnes); + MixedElemsPoison.push_back(ScalarOnes); + MixedElemsPoison.push_back(ScalarPoison); + Constant *VectorMixedPoison = ConstantVector::get(MixedElemsPoison); Value *Not = IRB.CreateXor(VectorZero, VectorOnes); Value *X; - EXPECT_TRUE(match(Not, m_Not(m_Value()))); - EXPECT_TRUE(match(Not, m_NotForbidUndef(m_Value(X)))); + EXPECT_TRUE(match(Not, m_Not(m_Value(X)))); EXPECT_TRUE(match(X, m_Zero())); Value *NotCommute = IRB.CreateXor(VectorOnes, VectorZero); Value *Y; - EXPECT_TRUE(match(NotCommute, m_Not(m_Value()))); - EXPECT_TRUE(match(NotCommute, m_NotForbidUndef(m_Value(Y)))); + EXPECT_TRUE(match(NotCommute, m_Not(m_Value(Y)))); EXPECT_TRUE(match(Y, m_Zero())); - Value *NotWithUndefs = IRB.CreateXor(VectorZero, VectorMixed); - EXPECT_TRUE(match(NotWithUndefs, m_Not(m_Value()))); - EXPECT_FALSE(match(NotWithUndefs, m_NotForbidUndef(m_Value()))); + Value *NotWithUndefs = IRB.CreateXor(VectorZero, VectorMixedUndef); + EXPECT_FALSE(match(NotWithUndefs, m_Not(m_Value()))); + + Value *NotWithPoisons = IRB.CreateXor(VectorZero, VectorMixedPoison); + EXPECT_TRUE(match(NotWithPoisons, m_Not(m_Value()))); + + Value *NotWithUndefsCommute = IRB.CreateXor(VectorMixedUndef, VectorZero); + EXPECT_FALSE(match(NotWithUndefsCommute, m_Not(m_Value()))); - Value *NotWithUndefsCommute = IRB.CreateXor(VectorMixed, VectorZero); - EXPECT_TRUE(match(NotWithUndefsCommute, m_Not(m_Value()))); - EXPECT_FALSE(match(NotWithUndefsCommute, m_NotForbidUndef(m_Value(X)))); + Value *NotWithPoisonsCommute = IRB.CreateXor(VectorMixedPoison, VectorZero); + EXPECT_TRUE(match(NotWithPoisonsCommute, m_Not(m_Value()))); } template struct MutableConstTest : PatternMatchTest { }; -- GitLab From 971237dab259bdaa619403fc6472bd1758d4dc18 Mon Sep 17 00:00:00 2001 From: jeanPerier Date: Wed, 17 Apr 2024 11:31:29 +0200 Subject: [PATCH 023/862] [flang] Retain internal and BIND(C) host procedure link in FIR (#87796) Currently, it is not possible to find back which fun.func is the host procedure of some internal procedure because the mangling of the internal procedure does not contain info about the BIND(C) name of the host. This info may be useful to ensure dwarf DW_TAG_subprogram of internal procedures are nested under DW_TAG_subprogram of host procedures for instance. --- flang/include/flang/Lower/CallInterface.h | 4 -- .../flang/Optimizer/Dialect/FIROpsSupport.h | 10 ++--- flang/lib/Lower/CallInterface.cpp | 40 +++++++++++++------ .../Optimizer/Transforms/ArrayValueCopy.cpp | 2 +- .../HLFIR/internal-procedures-bindc-host.f90 | 39 ++++++++++++++++++ .../test/Lower/HLFIR/internal-procedures.f90 | 6 +-- flang/test/Lower/OpenACC/acc-routine04.f90 | 2 +- .../threadprivate-host-association-2.f90 | 2 +- .../OpenMP/threadprivate-host-association.f90 | 2 +- flang/test/Lower/character-elemental.f90 | 15 ++++--- .../Lower/equivalence-with-host-assoc.f90 | 16 ++++---- .../Lower/explicit-interface-results-2.f90 | 4 +- .../test/Lower/host-associated-functions.f90 | 6 +-- flang/test/Lower/host-associated-globals.f90 | 6 +-- flang/test/Lower/host-associated.f90 | 32 +++++++-------- flang/test/Lower/polymorphic.f90 | 2 +- 16 files changed, 119 insertions(+), 69 deletions(-) create mode 100644 flang/test/Lower/HLFIR/internal-procedures-bindc-host.f90 diff --git a/flang/include/flang/Lower/CallInterface.h b/flang/include/flang/Lower/CallInterface.h index 80b057642537..a11e81b6593d 100644 --- a/flang/include/flang/Lower/CallInterface.h +++ b/flang/include/flang/Lower/CallInterface.h @@ -391,9 +391,6 @@ public: llvm_unreachable("getting host associated type in CallerInterface"); } - /// Set attributes on MLIR function. - void setFuncAttrs(mlir::func::FuncOp) const {} - private: /// Check that the input vector is complete. bool verifyActualInputs() const; @@ -444,7 +441,6 @@ public: bool hasHostAssociated() const; mlir::Type getHostAssociatedTy() const; mlir::Value getHostAssociatedTuple() const; - void setFuncAttrs(mlir::func::FuncOp) const; private: Fortran::lower::pft::FunctionLikeUnit &funit; diff --git a/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h b/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h index 3266ea3aa7fd..46b62d8de8d3 100644 --- a/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h +++ b/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h @@ -104,9 +104,9 @@ static constexpr llvm::StringRef getHostAssocAttrName() { return "fir.host_assoc"; } -/// Attribute to mark an internal procedure. -static constexpr llvm::StringRef getInternalProcedureAttrName() { - return "fir.internal_proc"; +/// Attribute to link an internal procedure to its host procedure symbol. +static constexpr llvm::StringRef getHostSymbolAttrName() { + return "fir.host_symbol"; } /// Attribute containing the original name of a function from before the @@ -122,8 +122,8 @@ bool hasHostAssociationArgument(mlir::func::FuncOp func); /// Is the function, \p func an internal procedure ? /// Some internal procedures may have access to saved host procedure /// variables even when they do not have a tuple argument. -inline bool isInternalPorcedure(mlir::func::FuncOp func) { - return func->hasAttr(fir::getInternalProcedureAttrName()); +inline bool isInternalProcedure(mlir::func::FuncOp func) { + return func->hasAttr(fir::getHostSymbolAttrName()); } /// Tell if \p value is: diff --git a/flang/lib/Lower/CallInterface.cpp b/flang/lib/Lower/CallInterface.cpp index 05a0c10c7097..2d4d17a2ef12 100644 --- a/flang/lib/Lower/CallInterface.cpp +++ b/flang/lib/Lower/CallInterface.cpp @@ -575,13 +575,6 @@ mlir::Value Fortran::lower::CalleeInterface::getHostAssociatedTuple() const { return converter.hostAssocTupleValue(); } -void Fortran::lower::CalleeInterface::setFuncAttrs( - mlir::func::FuncOp func) const { - if (funit.parentHasHostAssoc()) - func->setAttr(fir::getInternalProcedureAttrName(), - mlir::UnitAttr::get(func->getContext())); -} - //===----------------------------------------------------------------------===// // CallInterface implementation: this part is common to both caller and callee. //===----------------------------------------------------------------------===// @@ -589,6 +582,34 @@ void Fortran::lower::CalleeInterface::setFuncAttrs( static void addSymbolAttribute(mlir::func::FuncOp func, const Fortran::semantics::Symbol &sym, mlir::MLIRContext &mlirContext) { + const Fortran::semantics::Symbol &ultimate = sym.GetUltimate(); + // The link between an internal procedure and its host procedure is lost + // in FIR if the host is BIND(C) since the internal mangling will not + // allow retrieving the host bind(C) name, and therefore func.func symbol. + // Preserve it as an attribute so that this can be later retrieved. + if (Fortran::semantics::ClassifyProcedure(ultimate) == + Fortran::semantics::ProcedureDefinitionClass::Internal) { + if (ultimate.owner().kind() == + Fortran::semantics::Scope::Kind::Subprogram) { + if (const Fortran::semantics::Symbol *hostProcedure = + ultimate.owner().symbol()) { + std::string hostName = Fortran::lower::mangle::mangleName( + *hostProcedure, /*keepExternalInScope=*/true); + func->setAttr( + fir::getHostSymbolAttrName(), + mlir::SymbolRefAttr::get( + &mlirContext, mlir::StringAttr::get(&mlirContext, hostName))); + } + } else if (ultimate.owner().kind() == + Fortran::semantics::Scope::Kind::MainProgram) { + func->setAttr(fir::getHostSymbolAttrName(), + mlir::SymbolRefAttr::get( + &mlirContext, + mlir::StringAttr::get( + &mlirContext, fir::NameUniquer::doProgramEntry()))); + } + } + // Only add this on bind(C) functions for which the symbol is not reflected in // the current context. if (!Fortran::semantics::IsBindCProcedure(sym)) @@ -686,7 +707,6 @@ void Fortran::lower::CallInterface::declare() { for (const auto &placeHolder : llvm::enumerate(inputs)) if (!placeHolder.value().attributes.empty()) func.setArgAttrs(placeHolder.index(), placeHolder.value().attributes); - side().setFuncAttrs(func); setCUDAAttributes(func, side().getProcedureSymbol(), characteristic); } @@ -1599,10 +1619,6 @@ public: return proc; } - /// Set internal procedure attribute on MLIR function. Internal procedure - /// are defined in the current file and will not go through SignatureBuilder. - void setFuncAttrs(mlir::func::FuncOp) const {} - /// This is not the description of an indirect call. static constexpr bool isIndirectCall() { return false; } diff --git a/flang/lib/Optimizer/Transforms/ArrayValueCopy.cpp b/flang/lib/Optimizer/Transforms/ArrayValueCopy.cpp index 675314ed9da0..18ca5711bfea 100644 --- a/flang/lib/Optimizer/Transforms/ArrayValueCopy.cpp +++ b/flang/lib/Optimizer/Transforms/ArrayValueCopy.cpp @@ -728,7 +728,7 @@ conservativeCallConflict(llvm::ArrayRef reaches) { if (auto callee = call.getCallableForCallee().dyn_cast()) { auto module = op->getParentOfType(); - return isInternalPorcedure( + return isInternalProcedure( module.lookupSymbol(callee)); } return false; diff --git a/flang/test/Lower/HLFIR/internal-procedures-bindc-host.f90 b/flang/test/Lower/HLFIR/internal-procedures-bindc-host.f90 new file mode 100644 index 000000000000..07f60b98b094 --- /dev/null +++ b/flang/test/Lower/HLFIR/internal-procedures-bindc-host.f90 @@ -0,0 +1,39 @@ +! Test fir.host_sym attribute to retain link between internal +! and host procedure in FIR even when BIND(C) is involved. + +! RUN: bbc -emit-hlfir -o - %s | FileCheck %s +! RUN: bbc -emit-hlfir -o - %s | fir-opt -external-name-interop -o - |FileCheck %s --check-prefix=AFTER_RENAME_PASS + +subroutine foo() bind(c, name="some_c_name") + call bar() +contains + subroutine bar() + end subroutine +end subroutine +! CHECK: func.func @some_c_name() +! CHECK: func.func private @_QFfooPbar() attributes {fir.host_symbol = @some_c_name, llvm.linkage = #llvm.linkage} +! AFTER_RENAME_PASS: func.func @some_c_name() +! AFTER_RENAME_PASS: func.func private @_QFfooPbar() attributes {fir.host_symbol = @some_c_name, llvm.linkage = #llvm.linkage} + +subroutine notbindc() + call bar() +contains + subroutine bar() + end subroutine +end subroutine +! CHECK: func.func @_QPnotbindc() +! CHECK: func.func private @_QFnotbindcPbar() attributes {fir.host_symbol = @_QPnotbindc, llvm.linkage = #llvm.linkage} +! AFTER_RENAME_PASS: func.func @notbindc_() attributes {fir.internal_name = "_QPnotbindc"} +! AFTER_RENAME_PASS: func.func private @_QFnotbindcPbar() attributes {fir.host_symbol = @notbindc_, llvm.linkage = #llvm.linkage} + + +! Main program +call bar() +contains + subroutine bar() + end subroutine +end +! CHECK: func.func @_QQmain() +! CHECK: func.func private @_QFPbar() attributes {fir.host_symbol = @_QQmain, llvm.linkage = #llvm.linkage} +! AFTER_RENAME_PASS: func.func @_QQmain() +! AFTER_RENAME_PASS: func.func private @_QFPbar() attributes {fir.host_symbol = @_QQmain, llvm.linkage = #llvm.linkage} diff --git a/flang/test/Lower/HLFIR/internal-procedures.f90 b/flang/test/Lower/HLFIR/internal-procedures.f90 index c898903b6fbe..fff7125897dd 100644 --- a/flang/test/Lower/HLFIR/internal-procedures.f90 +++ b/flang/test/Lower/HLFIR/internal-procedures.f90 @@ -10,7 +10,7 @@ subroutine internal end subroutine end subroutine ! CHECK-LABEL: func.func private @_QFtest_explicit_shape_arrayPinternal( -! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>>> {fir.host_assoc}) attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>>> {fir.host_assoc}) attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { ! CHECK: %[[VAL_1:.*]] = arith.constant 0 : i32 ! CHECK: %[[VAL_2:.*]] = fir.coordinate_of %[[VAL_0]], %[[VAL_1]] : (!fir.ref>>>, i32) -> !fir.ref>> ! CHECK: %[[VAL_3:.*]] = fir.load %[[VAL_2]] : !fir.ref>> @@ -28,7 +28,7 @@ subroutine internal end subroutine end subroutine ! CHECK-LABEL: func.func private @_QFtest_assumed_shapePinternal( -! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>>> {fir.host_assoc}) attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>>> {fir.host_assoc}) attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { ! CHECK: %[[VAL_1:.*]] = arith.constant 0 : i32 ! CHECK: %[[VAL_2:.*]] = fir.coordinate_of %[[VAL_0]], %[[VAL_1]] : (!fir.ref>>>, i32) -> !fir.ref>> ! CHECK: %[[VAL_3:.*]] = fir.load %[[VAL_2]] : !fir.ref>> @@ -45,7 +45,7 @@ subroutine internal() end subroutine end subroutine ! CHECK-LABEL: func.func private @_QFtest_scalar_charPinternal( -! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>> {fir.host_assoc}) attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>> {fir.host_assoc}) attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { ! CHECK: %[[VAL_1:.*]] = arith.constant 0 : i32 ! CHECK: %[[VAL_2:.*]] = fir.coordinate_of %[[VAL_0]], %[[VAL_1]] : (!fir.ref>>, i32) -> !fir.ref> ! CHECK: %[[VAL_3:.*]] = fir.load %[[VAL_2]] : !fir.ref> diff --git a/flang/test/Lower/OpenACC/acc-routine04.f90 b/flang/test/Lower/OpenACC/acc-routine04.f90 index 2339c23eaaf8..f60337616390 100644 --- a/flang/test/Lower/OpenACC/acc-routine04.f90 +++ b/flang/test/Lower/OpenACC/acc-routine04.f90 @@ -31,4 +31,4 @@ end program ! CHECK: acc.routine @acc_routine_0 func(@_QMdummy_modPsub1) seq ! CHECK: func.func @_QMdummy_modPsub1(%arg0: !fir.ref {fir.bindc_name = "i"}) attributes {acc.routine_info = #acc.routine_info<[@acc_routine_0]>} ! CHECK: func.func @_QQmain() attributes {fir.bindc_name = "test_acc_routine"} -! CHECK: func.func private @_QFPsub2() attributes {acc.routine_info = #acc.routine_info<[@acc_routine_1]>, llvm.linkage = #llvm.linkage} +! CHECK: func.func private @_QFPsub2() attributes {acc.routine_info = #acc.routine_info<[@acc_routine_1]>, fir.host_symbol = @_QQmain, llvm.linkage = #llvm.linkage} diff --git a/flang/test/Lower/OpenMP/threadprivate-host-association-2.f90 b/flang/test/Lower/OpenMP/threadprivate-host-association-2.f90 index b47bff5bebb0..a8d29baf74f2 100644 --- a/flang/test/Lower/OpenMP/threadprivate-host-association-2.f90 +++ b/flang/test/Lower/OpenMP/threadprivate-host-association-2.f90 @@ -12,7 +12,7 @@ !CHECK: fir.call @_QFPsub() fastmath : () -> () !CHECK: return !CHECK: } -!CHECK: func.func private @_QFPsub() attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { +!CHECK: func.func private @_QFPsub() attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { !CHECK: %[[A:.*]] = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFEa"} !CHECK: %[[A_DECL:.*]]:2 = hlfir.declare %[[A]] {uniq_name = "_QFEa"} : (!fir.ref) -> (!fir.ref, !fir.ref) !CHECK: %[[A_ADDR:.*]] = fir.address_of(@_QFEa) : !fir.ref diff --git a/flang/test/Lower/OpenMP/threadprivate-host-association.f90 b/flang/test/Lower/OpenMP/threadprivate-host-association.f90 index 98f7b51bb971..096e510c19c6 100644 --- a/flang/test/Lower/OpenMP/threadprivate-host-association.f90 +++ b/flang/test/Lower/OpenMP/threadprivate-host-association.f90 @@ -11,7 +11,7 @@ !CHECK: fir.call @_QFPsub() fastmath : () -> () !CHECK: return !CHECK: } -!CHECK: func.func private @_QFPsub() attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { +!CHECK: func.func private @_QFPsub() attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { !CHECK: %[[A:.*]] = fir.address_of(@_QFEa) : !fir.ref !CHECK: %[[A_DECL:.*]]:2 = hlfir.declare %[[A]] {uniq_name = "_QFEa"} : (!fir.ref) -> (!fir.ref, !fir.ref) !CHECK: %[[TP_A:.*]] = omp.threadprivate %[[A_DECL]]#1 : !fir.ref -> !fir.ref diff --git a/flang/test/Lower/character-elemental.f90 b/flang/test/Lower/character-elemental.f90 index 6c46454176f5..9a9cf8bf2d9c 100644 --- a/flang/test/Lower/character-elemental.f90 +++ b/flang/test/Lower/character-elemental.f90 @@ -5,6 +5,12 @@ subroutine substring_main character*7 :: string(2) = ['12 ', '12 '] integer :: result(2) integer :: ival +interface + elemental function inner(arg) + character(len=*), intent(in) :: arg + integer :: inner + end function inner +end interface ival = 1 ! CHECK: %[[a0:.*]] = fir.alloca i32 {bindc_name = "ival", uniq_name = "_QFsubstring_mainEival"} @@ -26,14 +32,7 @@ subroutine substring_main ! CHECK: %[[a14:.*]] = fir.coordinate_of %[[a13]], %[[a12]] : (!fir.ref>>, index) -> !fir.ref> ! CHECK: %[[a15:.*]] = fir.convert %[[a14]] : (!fir.ref>) -> !fir.ref> ! CHECK: %[[a16:.*]] = fir.emboxchar %[[a15]], {{.*}} : (!fir.ref>, index) -> !fir.boxchar<1> - ! CHECK: %[[a17:.*]] = fir.call @_QFsubstring_mainPinner(%[[a16]]) {{.*}}: (!fir.boxchar<1>) -> i32 + ! CHECK: %[[a17:.*]] = fir.call @_QPinner(%[[a16]]) {{.*}}: (!fir.boxchar<1>) -> i32 result = inner(string(1:2)(ival:ival)) print *, result -contains - elemental function inner(arg) - character(len=*), intent(in) :: arg - integer :: inner - - inner = len(arg) - end function inner end subroutine substring_main diff --git a/flang/test/Lower/equivalence-with-host-assoc.f90 b/flang/test/Lower/equivalence-with-host-assoc.f90 index 0ffb1bc5bf9e..b8ce72f3787c 100644 --- a/flang/test/Lower/equivalence-with-host-assoc.f90 +++ b/flang/test/Lower/equivalence-with-host-assoc.f90 @@ -10,7 +10,7 @@ contains i1 = j1 end subroutine inner end subroutine test1 -! FIR-LABEL: func.func private @_QFtest1Pinner() attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { +! FIR-LABEL: func.func private @_QFtest1Pinner() attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { ! FIR: %[[VAL_0:.*]] = fir.address_of(@_QFtest1Ei1) : !fir.ref> ! FIR: %[[VAL_1:.*]] = fir.convert %[[VAL_0]] : (!fir.ref>) -> !fir.ref> ! FIR: %[[VAL_2:.*]] = arith.constant 0 : index @@ -24,7 +24,7 @@ end subroutine test1 ! FIR: return ! FIR: } -! HLFIR-LABEL: func.func private @_QFtest1Pinner() attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { +! HLFIR-LABEL: func.func private @_QFtest1Pinner() attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { ! HLFIR: %[[VAL_0:.*]] = fir.address_of(@_QFtest1Ei1) : !fir.ref> ! HLFIR: %[[VAL_1:.*]] = fir.convert %[[VAL_0]] : (!fir.ref>) -> !fir.ref> ! HLFIR: %[[VAL_2:.*]] = arith.constant 0 : index @@ -54,7 +54,7 @@ contains end subroutine inner end subroutine host end module test2 -! FIR-LABEL: func.func private @_QMtest2FhostPinner() attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { +! FIR-LABEL: func.func private @_QMtest2FhostPinner() attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { ! FIR: %[[VAL_0:.*]] = fir.address_of(@_QMtest2FhostEf1) : !fir.ref> ! FIR: %[[VAL_1:.*]] = fir.convert %[[VAL_0]] : (!fir.ref>) -> !fir.ref> ! FIR: %[[VAL_2:.*]] = arith.constant 0 : index @@ -68,7 +68,7 @@ end module test2 ! FIR: return ! FIR: } -! HLFIR-LABEL: func.func private @_QMtest2FhostPinner() attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { +! HLFIR-LABEL: func.func private @_QMtest2FhostPinner() attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { ! HLFIR: %[[VAL_0:.*]] = fir.address_of(@_QMtest2FhostEf1) : !fir.ref> ! HLFIR: %[[VAL_1:.*]] = fir.convert %[[VAL_0]] : (!fir.ref>) -> !fir.ref> ! HLFIR: %[[VAL_2:.*]] = arith.constant 0 : index @@ -94,7 +94,7 @@ contains i1 = j1 + k1 end subroutine inner end subroutine test3 -! FIR-LABEL: func.func private @_QFtest3Pinner() attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { +! FIR-LABEL: func.func private @_QFtest3Pinner() attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { ! FIR: %[[VAL_0:.*]] = fir.address_of(@blk_) : !fir.ref> ! FIR: %[[VAL_1:.*]] = fir.convert %[[VAL_0]] : (!fir.ref>) -> !fir.ref> ! FIR: %[[VAL_2:.*]] = arith.constant 0 : index @@ -115,7 +115,7 @@ end subroutine test3 ! FIR: return ! FIR: } -! HLFIR-LABEL: func.func private @_QFtest3Pinner() attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { +! HLFIR-LABEL: func.func private @_QFtest3Pinner() attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { ! HLFIR: %[[VAL_0:.*]] = fir.address_of(@blk_) : !fir.ref> ! HLFIR: %[[VAL_1:.*]] = fir.convert %[[VAL_0]] : (!fir.ref>) -> !fir.ref> ! HLFIR: %[[VAL_2:.*]] = arith.constant 0 : index @@ -149,7 +149,7 @@ contains i1 = j1 + k1 end subroutine inner end subroutine test4 -! FIR-LABEL: func.func private @_QFtest4Pinner() attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { +! FIR-LABEL: func.func private @_QFtest4Pinner() attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { ! FIR: %[[VAL_0:.*]] = fir.address_of(@blk_) : !fir.ref> ! FIR: %[[VAL_1:.*]] = fir.convert %[[VAL_0]] : (!fir.ref>) -> !fir.ref> ! FIR: %[[VAL_2:.*]] = arith.constant 0 : index @@ -170,7 +170,7 @@ end subroutine test4 ! FIR: return ! FIR: } -! HLFIR-LABEL: func.func private @_QFtest4Pinner() attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { +! HLFIR-LABEL: func.func private @_QFtest4Pinner() attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { ! HLFIR: %[[VAL_0:.*]] = fir.address_of(@blk_) : !fir.ref> ! HLFIR: %[[VAL_1:.*]] = fir.convert %[[VAL_0]] : (!fir.ref>) -> !fir.ref> ! HLFIR: %[[VAL_2:.*]] = arith.constant 0 : index diff --git a/flang/test/Lower/explicit-interface-results-2.f90 b/flang/test/Lower/explicit-interface-results-2.f90 index 86aae720e7fc..a63ee5fc9179 100644 --- a/flang/test/Lower/explicit-interface-results-2.f90 +++ b/flang/test/Lower/explicit-interface-results-2.f90 @@ -70,7 +70,7 @@ subroutine host4() call internal_proc_a() contains ! CHECK-LABEL: func private @_QFhost4Pinternal_proc_a -! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>> {fir.host_assoc}) attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>> {fir.host_assoc}) attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { subroutine internal_proc_a() call takes_array(return_array()) ! CHECK: %[[VAL_1:.*]] = arith.constant 0 : i32 @@ -94,7 +94,7 @@ subroutine host5() implicit none call internal_proc_a() contains -! CHECK-LABEL: func private @_QFhost5Pinternal_proc_a() attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { +! CHECK-LABEL: func private @_QFhost5Pinternal_proc_a() attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { subroutine internal_proc_a() call takes_array(return_array()) ! CHECK: %[[VAL_0:.*]] = fir.address_of(@_QMsome_moduleEn_module) : !fir.ref diff --git a/flang/test/Lower/host-associated-functions.f90 b/flang/test/Lower/host-associated-functions.f90 index 78d081748c2f..d67a74fa3998 100644 --- a/flang/test/Lower/host-associated-functions.f90 +++ b/flang/test/Lower/host-associated-functions.f90 @@ -20,7 +20,7 @@ subroutine capture_char_func_dummy(char_func_dummy, n) call internal() contains ! CHECK-LABEL: func private @_QFcapture_char_func_dummyPinternal( - ! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref ()>, i64>, !fir.ref>> {fir.host_assoc}) attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { + ! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref ()>, i64>, !fir.ref>> {fir.host_assoc}) attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { subroutine internal() ! CHECK: %[[VAL_1:.*]] = arith.constant 0 : i32 ! CHECK: %[[VAL_2:.*]] = fir.coordinate_of %[[VAL_0]], %[[VAL_1]] : (!fir.ref ()>, i64>, !fir.ref>>, i32) -> !fir.ref ()>, i64>> @@ -56,7 +56,7 @@ subroutine capture_char_func_assumed_dummy(char_func_dummy) call internal() contains ! CHECK-LABEL: func private @_QFcapture_char_func_assumed_dummyPinternal( -! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref ()>, i64>>> {fir.host_assoc}) attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref ()>, i64>>> {fir.host_assoc}) attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { subroutine internal() ! CHECK: %[[VAL_1:.*]] = arith.constant 0 : i32 ! CHECK: %[[VAL_2:.*]] = fir.coordinate_of %[[VAL_0]], %[[VAL_1]] : (!fir.ref ()>, i64>>>, i32) -> !fir.ref ()>, i64>> @@ -110,7 +110,7 @@ subroutine capture_array_func(n) contains subroutine internal() ! CHECK-LABEL: func private @_QFcapture_array_funcPinternal( -! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>> {fir.host_assoc}) attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>> {fir.host_assoc}) attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { ! CHECK: %[[VAL_1:.*]] = arith.constant 0 : i32 ! CHECK: %[[VAL_2:.*]] = fir.coordinate_of %[[VAL_0]], %[[VAL_1]] : (!fir.ref>>, i32) -> !fir.llvm_ptr> ! CHECK: %[[VAL_3:.*]] = fir.load %[[VAL_2]] : !fir.llvm_ptr> diff --git a/flang/test/Lower/host-associated-globals.f90 b/flang/test/Lower/host-associated-globals.f90 index fe612e777aea..c91a5a46af0d 100644 --- a/flang/test/Lower/host-associated-globals.f90 +++ b/flang/test/Lower/host-associated-globals.f90 @@ -37,7 +37,7 @@ contains print *, j_in_equiv, not_in_equiv end subroutine end subroutine -! CHECK-LABEL: func.func private @_QFtest_commonPbar() attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { +! CHECK-LABEL: func.func private @_QFtest_commonPbar() attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { ! CHECK: %[[VAL_0:.*]] = fir.address_of(@x_) : !fir.ref> ! CHECK: %[[VAL_1:.*]] = fir.convert %[[VAL_0]] : (!fir.ref>) -> !fir.ref> ! CHECK: %[[VAL_2:.*]] = arith.constant 4 : index @@ -59,7 +59,7 @@ contains print *, j_in_equiv, not_in_equiv end subroutine end subroutine -! CHECK-LABEL: func.func private @_QFsaved_equivPbar() attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { +! CHECK-LABEL: func.func private @_QFsaved_equivPbar() attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { ! CHECK: %[[VAL_0:.*]] = fir.address_of(@_QFsaved_equivEi) : !fir.ref> ! CHECK: %[[VAL_1:.*]] = arith.constant 4 : index ! CHECK: %[[VAL_2:.*]] = fir.coordinate_of %[[VAL_0]], %[[VAL_1]] : (!fir.ref>, index) -> !fir.ref @@ -80,7 +80,7 @@ contains end subroutine end subroutine ! CHECK-LABEL: func.func private @_QFmixed_capturePbar( -! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>> {fir.host_assoc}) attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>> {fir.host_assoc}) attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { ! CHECK: %[[VAL_1:.*]] = fir.address_of(@_QFmixed_captureEsaved_i) : !fir.ref> ! CHECK: %[[VAL_2:.*]] = arith.constant 0 : index ! CHECK: %[[VAL_3:.*]] = fir.coordinate_of %[[VAL_1]], %[[VAL_2]] : (!fir.ref>, index) -> !fir.ref diff --git a/flang/test/Lower/host-associated.f90 b/flang/test/Lower/host-associated.f90 index f88903c8af80..cdc7e6a05288 100644 --- a/flang/test/Lower/host-associated.f90 +++ b/flang/test/Lower/host-associated.f90 @@ -20,7 +20,7 @@ subroutine test1 print *, i contains ! CHECK-LABEL: func private @_QFtest1Ptest1_internal( - ! CHECK-SAME: %[[arg:[^:]*]]: !fir.ref>> {fir.host_assoc}) attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { + ! CHECK-SAME: %[[arg:[^:]*]]: !fir.ref>> {fir.host_assoc}) attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { ! CHECK: %[[iaddr:.*]] = fir.coordinate_of %[[arg]], %c0 ! CHECK: %[[i:.*]] = fir.load %[[iaddr]] : !fir.llvm_ptr> ! CHECK: %[[val:.*]] = fir.call @_QPifoo() {{.*}}: () -> i32 @@ -47,7 +47,7 @@ subroutine test2 print *, a, b contains ! CHECK-LABEL: func private @_QFtest2Ptest2_internal( - ! CHECK-SAME: %[[arg:[^:]*]]: !fir.ref, !fir.ref>> {fir.host_assoc}) attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { + ! CHECK-SAME: %[[arg:[^:]*]]: !fir.ref, !fir.ref>> {fir.host_assoc}) attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { subroutine test2_internal ! CHECK: %[[a:.*]] = fir.coordinate_of %[[arg]], %c0 ! CHECK: %[[aa:.*]] = fir.load %[[a]] : !fir.llvm_ptr> @@ -62,7 +62,7 @@ contains end subroutine test2_internal ! CHECK-LABEL: func private @_QFtest2Ptest2_inner( - ! CHECK-SAME: %[[arg:[^:]*]]: !fir.ref, !fir.ref>> {fir.host_assoc}) attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { + ! CHECK-SAME: %[[arg:[^:]*]]: !fir.ref, !fir.ref>> {fir.host_assoc}) attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { subroutine test2_inner ! CHECK: %[[a:.*]] = fir.coordinate_of %[[arg]], %c0 ! CHECK: %[[aa:.*]] = fir.load %[[a]] : !fir.llvm_ptr> @@ -96,7 +96,7 @@ subroutine test6(c) contains ! CHECK-LABEL: func private @_QFtest6Ptest6_inner( - ! CHECK-SAME: %[[tup:.*]]: !fir.ref>> {fir.host_assoc}) attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { + ! CHECK-SAME: %[[tup:.*]]: !fir.ref>> {fir.host_assoc}) attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { subroutine test6_inner ! CHECK: %[[coor:.*]] = fir.coordinate_of %[[tup]], %c0{{.*}} : (!fir.ref>>, i32) -> !fir.ref> ! CHECK: %[[load:.*]] = fir.load %[[coor]] : !fir.ref> @@ -138,7 +138,7 @@ subroutine test3(p,q,i) contains ! CHECK-LABEL: func private @_QFtest3Ptest3_inner( - ! CHECK-SAME: %[[tup:.*]]: !fir.ref>, !fir.box>>> {fir.host_assoc}) attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { + ! CHECK-SAME: %[[tup:.*]]: !fir.ref>, !fir.box>>> {fir.host_assoc}) attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { subroutine test3_inner ! CHECK: %[[pcoor:.*]] = fir.coordinate_of %[[tup]], %c0{{.*}} : (!fir.ref>, !fir.box>>>, i32) -> !fir.ref>> ! CHECK: %[[p:.*]] = fir.load %[[pcoor]] : !fir.ref>> @@ -185,7 +185,7 @@ subroutine test3a(p) contains ! CHECK: func private @_QFtest3aPtest3a_inner( - ! CHECK-SAME: %[[tup:.*]]: !fir.ref>, !fir.box>>> {fir.host_assoc}) attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { + ! CHECK-SAME: %[[tup:.*]]: !fir.ref>, !fir.box>>> {fir.host_assoc}) attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { subroutine test3a_inner ! CHECK: %[[pcoor:.*]] = fir.coordinate_of %[[tup]], %c0{{.*}} : (!fir.ref>, !fir.box>>>, i32) -> !fir.ref>> ! CHECK: %[[p:.*]] = fir.load %[[pcoor]] : !fir.ref>> @@ -229,7 +229,7 @@ subroutine test4 contains ! CHECK-LABEL: func private @_QFtest4Ptest4_inner( - ! CHECK-SAME:%[[tup:.*]]: !fir.ref>>, !fir.ref>>>> {fir.host_assoc}) attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { + ! CHECK-SAME:%[[tup:.*]]: !fir.ref>>, !fir.ref>>>> {fir.host_assoc}) attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { subroutine test4_inner ! CHECK: %[[ptup:.*]] = fir.coordinate_of %[[tup]], %c0{{.*}} : (!fir.ref>>, !fir.ref>>>>, i32) -> !fir.llvm_ptr>>> ! CHECK: %[[p:.*]] = fir.load %[[ptup]] : !fir.llvm_ptr>>> @@ -271,7 +271,7 @@ subroutine test5 contains ! CHECK-LABEL: func private @_QFtest5Ptest5_inner( - ! CHECK-SAME:%[[tup:.*]]: !fir.ref>>>, !fir.ref>>>>> {fir.host_assoc}) attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { + ! CHECK-SAME:%[[tup:.*]]: !fir.ref>>>, !fir.ref>>>>> {fir.host_assoc}) attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { subroutine test5_inner ! CHECK: %[[ptup:.*]] = fir.coordinate_of %[[tup]], %c0{{.*}} : (!fir.ref>>>, !fir.ref>>>>>, i32) -> !fir.llvm_ptr>>>> ! CHECK: %[[p:.*]] = fir.load %[[ptup]] : !fir.llvm_ptr>>>> @@ -309,7 +309,7 @@ subroutine test7(j, k) contains ! CHECK-LABEL: func private @_QFtest7Ptest7_inner( -! CHECK-SAME: %[[i:.*]]: !fir.ref{{.*}}, %[[tup:.*]]: !fir.ref>> {fir.host_assoc}) -> i32 attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { +! CHECK-SAME: %[[i:.*]]: !fir.ref{{.*}}, %[[tup:.*]]: !fir.ref>> {fir.host_assoc}) -> i32 attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { elemental integer function test7_inner(i) implicit none integer, intent(in) :: i @@ -330,7 +330,7 @@ subroutine issue990() call bar() contains ! CHECK-LABEL: func private @_QFissue990Pbar( -! CHECK-SAME: %[[tup:.*]]: !fir.ref>> {fir.host_assoc}) attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { +! CHECK-SAME: %[[tup:.*]]: !fir.ref>> {fir.host_assoc}) attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { subroutine bar() integer :: stmt_func, i stmt_func(i) = i + captured @@ -352,7 +352,7 @@ subroutine issue990b() call bar() contains ! CHECK-LABEL: func private @_QFissue990bPbar( -! CHECK-SAME: %[[tup:.*]]: !fir.ref>> {fir.host_assoc}) attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { +! CHECK-SAME: %[[tup:.*]]: !fir.ref>> {fir.host_assoc}) attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { subroutine bar() ! CHECK: %[[tupAddr:.*]] = fir.coordinate_of %[[tup]], %c0{{.*}} : (!fir.ref>>, i32) -> !fir.llvm_ptr> ! CHECK: %[[addr:.*]] = fir.load %[[tupAddr]] : !fir.llvm_ptr> @@ -373,7 +373,7 @@ subroutine test8(dummy_proc) call bar() contains ! CHECK-LABEL: func private @_QFtest8Pbar( -! CHECK-SAME: %[[tup:.*]]: !fir.ref ()>>> {fir.host_assoc}) attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { +! CHECK-SAME: %[[tup:.*]]: !fir.ref ()>>> {fir.host_assoc}) attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { subroutine bar() ! CHECK: %[[tupAddr:.*]] = fir.coordinate_of %[[tup]], %c0{{.*}} : (!fir.ref ()>>>, i32) -> !fir.ref ()>> ! CHECK: %[[dummyProc:.*]] = fir.load %[[tupAddr]] : !fir.ref ()>> @@ -393,7 +393,7 @@ subroutine test9(dummy_proc) call bar() contains ! CHECK-LABEL: func private @_QFtest9Pbar( -! CHECK-SAME: %[[tup:.*]]: !fir.ref ()>>> {fir.host_assoc}) attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { +! CHECK-SAME: %[[tup:.*]]: !fir.ref ()>>> {fir.host_assoc}) attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { subroutine bar() ! CHECK: %[[tupAddr:.*]] = fir.coordinate_of %[[tup]], %c0{{.*}} : (!fir.ref ()>>>, i32) -> !fir.ref ()>> ! CHECK: %[[dummyProc:.*]] = fir.load %[[tupAddr]] : !fir.ref ()>> @@ -416,7 +416,7 @@ subroutine test10(i) call bar() contains ! CHECK-LABEL: func private @_QFtest10Pbar( -! CHECK-SAME: %[[tup:.*]]: !fir.ref>>>>> {fir.host_assoc}) attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { +! CHECK-SAME: %[[tup:.*]]: !fir.ref>>>>> {fir.host_assoc}) attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { subroutine bar() ! CHECK: %[[tupAddr:.*]] = fir.coordinate_of %[[tup]], %c0{{.*}} : (!fir.ref>>>>>, i32) -> !fir.llvm_ptr>>>> ! CHECK: fir.load %[[tupAddr]] : !fir.llvm_ptr>>>> @@ -435,7 +435,7 @@ end subroutine ! CHECK-LABEL: func private @_QFtest_proc_dummyPtest_proc_dummy_a( ! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref {fir.bindc_name = "j"}, -! CHECK-SAME: %[[VAL_1:.*]]: !fir.ref>> {fir.host_assoc}) attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { +! CHECK-SAME: %[[VAL_1:.*]]: !fir.ref>> {fir.host_assoc}) attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { ! CHECK: %[[VAL_2:.*]] = arith.constant 0 : i32 ! CHECK: %[[VAL_3:.*]] = fir.coordinate_of %[[VAL_1]], %[[VAL_2]] : (!fir.ref>>, i32) -> !fir.llvm_ptr> ! CHECK: %[[VAL_4:.*]] = fir.load %[[VAL_3]] : !fir.llvm_ptr> @@ -528,7 +528,7 @@ end subroutine test_proc_dummy_other ! CHECK-LABEL: func private @_QFtest_proc_dummy_charPgen_message( ! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>, ! CHECK-SAME: %[[VAL_1:.*]]: index, -! CHECK-SAME: %[[VAL_2:.*]]: !fir.ref>> {fir.host_assoc}) -> !fir.boxchar<1> attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { +! CHECK-SAME: %[[VAL_2:.*]]: !fir.ref>> {fir.host_assoc}) -> !fir.boxchar<1> attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { ! CHECK-DAG: %[[VAL_3:.*]] = arith.constant 0 : i32 ! CHECK-DAG: %[[VAL_4:.*]] = arith.constant 10 : index ! CHECK-DAG: %[[VAL_5:.*]] = arith.constant false diff --git a/flang/test/Lower/polymorphic.f90 b/flang/test/Lower/polymorphic.f90 index e031b4805dc5..70c1f768e389 100644 --- a/flang/test/Lower/polymorphic.f90 +++ b/flang/test/Lower/polymorphic.f90 @@ -520,7 +520,7 @@ module polymorphic_test end subroutine ! CHECK-LABEL: func.func private @_QMpolymorphic_testFhost_assocPinternal( -! CHECK-SAME: %[[TUPLE:.*]]: !fir.ref>>> {fir.host_assoc}) attributes {fir.internal_proc, llvm.linkage = #llvm.linkage} { +! CHECK-SAME: %[[TUPLE:.*]]: !fir.ref>>> {fir.host_assoc}) attributes {fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage} { ! CHECK: %[[POS_IN_TUPLE:.*]] = arith.constant 0 : i32 ! CHECK: %[[COORD_OF_CLASS:.*]] = fir.coordinate_of %[[TUPLE]], %[[POS_IN_TUPLE]] : (!fir.ref>>>, i32) -> !fir.ref>> ! CHECK: %[[CLASS:.*]] = fir.load %[[COORD_OF_CLASS]] : !fir.ref>> -- GitLab From b512df660ef136f8bbd0895bf862a827923a6714 Mon Sep 17 00:00:00 2001 From: Vyacheslav Levytskyy Date: Wed, 17 Apr 2024 11:50:37 +0200 Subject: [PATCH 024/862] [SPIR-V] Improve Tablegen instruction selection and account for a pointer size of the target (#88725) This PR resolves the issue that SPIR-V Backend uses the notion of a pointer size of the target, most notably, in legalizer code, but Tablegen instruction selection in SPIR-V Backend doesn't account for a pointer size of the target. See https://github.com/llvm/llvm-project/issues/88723 for a detailed description. There are 3 test cases attached to the PR that reproduced the issue, when dealing with spirv32-spirv64 differences, and are working correctly now with this PR. --- llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp | 3 +- llvm/lib/Target/SPIRV/SPIRVInstrInfo.cpp | 6 +- llvm/lib/Target/SPIRV/SPIRVInstrInfo.td | 18 ++++-- .../Target/SPIRV/SPIRVInstructionSelector.cpp | 8 ++- llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp | 5 +- llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp | 62 ++++++++++++++----- .../Target/SPIRV/SPIRVRegisterBankInfo.cpp | 20 +----- llvm/lib/Target/SPIRV/SPIRVRegisterBanks.td | 7 +-- llvm/lib/Target/SPIRV/SPIRVRegisterInfo.td | 49 ++++++++------- .../CodeGen/SPIRV/instructions/select-phi.ll | 4 ++ .../SPIRV/instructions/select-ptr-load.ll | 25 ++++++++ .../test/CodeGen/SPIRV/instructions/select.ll | 3 + .../SPIRV/{select.ll => select-builtin.ll} | 2 + 13 files changed, 138 insertions(+), 74 deletions(-) create mode 100644 llvm/test/CodeGen/SPIRV/instructions/select-ptr-load.ll rename llvm/test/CodeGen/SPIRV/{select.ll => select-builtin.ll} (67%) diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp index 05e41e06248e..cebe230d3e8c 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp @@ -653,7 +653,8 @@ Register SPIRVGlobalRegistry::buildGlobalVariable( auto MRI = MIRBuilder.getMRI(); assert(MRI->getType(ResVReg).isPointer() && "Pointer type is expected"); if (Reg != ResVReg) { - LLT RegLLTy = LLT::pointer(MRI->getType(ResVReg).getAddressSpace(), 32); + LLT RegLLTy = + LLT::pointer(MRI->getType(ResVReg).getAddressSpace(), getPointerSize()); MRI->setType(Reg, RegLLTy); assignSPIRVTypeToVReg(BaseType, Reg, MIRBuilder.getMF()); } else { diff --git a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.cpp b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.cpp index aacfecc1e313..af98f2f88045 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.cpp @@ -247,8 +247,10 @@ void SPIRVInstrInfo::copyPhysReg(MachineBasicBlock &MBB, bool SPIRVInstrInfo::expandPostRAPseudo(MachineInstr &MI) const { if (MI.getOpcode() == SPIRV::GET_ID || MI.getOpcode() == SPIRV::GET_fID || - MI.getOpcode() == SPIRV::GET_pID || MI.getOpcode() == SPIRV::GET_vfID || - MI.getOpcode() == SPIRV::GET_vID || MI.getOpcode() == SPIRV::GET_vpID) { + MI.getOpcode() == SPIRV::GET_pID32 || + MI.getOpcode() == SPIRV::GET_pID64 || MI.getOpcode() == SPIRV::GET_vfID || + MI.getOpcode() == SPIRV::GET_vID || MI.getOpcode() == SPIRV::GET_vpID32 || + MI.getOpcode() == SPIRV::GET_vpID64) { auto &MRI = MI.getMF()->getRegInfo(); MRI.replaceRegWith(MI.getOperand(0).getReg(), MI.getOperand(1).getReg()); MI.eraseFromParent(); diff --git a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td index a3f981457c8d..151d0ec1fe56 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td +++ b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td @@ -19,10 +19,12 @@ let isCodeGenOnly=1 in { def DECL_TYPE: Pseudo<(outs ANYID:$dst_id), (ins ANYID:$src_id, TYPE:$src_ty)>; def GET_ID: Pseudo<(outs ID:$dst_id), (ins ANYID:$src)>; def GET_fID: Pseudo<(outs fID:$dst_id), (ins ANYID:$src)>; - def GET_pID: Pseudo<(outs pID:$dst_id), (ins ANYID:$src)>; + def GET_pID32: Pseudo<(outs pID32:$dst_id), (ins ANYID:$src)>; + def GET_pID64: Pseudo<(outs pID64:$dst_id), (ins ANYID:$src)>; def GET_vID: Pseudo<(outs vID:$dst_id), (ins ANYID:$src)>; def GET_vfID: Pseudo<(outs vfID:$dst_id), (ins ANYID:$src)>; - def GET_vpID: Pseudo<(outs vpID:$dst_id), (ins ANYID:$src)>; + def GET_vpID32: Pseudo<(outs vpID32:$dst_id), (ins ANYID:$src)>; + def GET_vpID64: Pseudo<(outs vpID64:$dst_id), (ins ANYID:$src)>; } def SPVTypeBin : SDTypeProfile<1, 2, []>; @@ -66,8 +68,10 @@ multiclass TernOpTypedGen opCode, SDNode node, bit genP = def SIVCond: TernOpTyped; } if genP then { - def SPSCond: TernOpTyped; - def SPVCond: TernOpTyped; + def SPSCond32: TernOpTyped; + def SPVCond32: TernOpTyped; + def SPSCond64: TernOpTyped; + def SPVCond64: TernOpTyped; } if genV then { if genF then { @@ -79,8 +83,10 @@ multiclass TernOpTypedGen opCode, SDNode node, bit genP = def VIVCond: TernOpTyped; } if genP then { - def VPSCond: TernOpTyped; - def VPVCond: TernOpTyped; + def VPSCond32: TernOpTyped; + def VPVCond32: TernOpTyped; + def VPSCond64: TernOpTyped; + def VPVCond64: TernOpTyped; } } } diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp index 200fe38298c0..72e5a7bcac98 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp @@ -290,14 +290,18 @@ bool SPIRVInstructionSelector::select(MachineInstr &I) { // If it's not a GMIR instruction, we've selected it already. if (!isPreISelGenericOpcode(Opcode)) { if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more. - auto *Def = MRI->getVRegDef(I.getOperand(1).getReg()); + Register DstReg = I.getOperand(0).getReg(); + Register SrcReg = I.getOperand(1).getReg(); + auto *Def = MRI->getVRegDef(SrcReg); if (isTypeFoldingSupported(Def->getOpcode())) { + if (MRI->getType(DstReg).isPointer()) + MRI->setType(DstReg, LLT::scalar(32)); bool Res = selectImpl(I, *CoverageInfo); assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT); if (Res) return Res; } - MRI->replaceRegWith(I.getOperand(1).getReg(), I.getOperand(0).getReg()); + MRI->replaceRegWith(SrcReg, DstReg); I.removeFromParent(); return true; } else if (I.getNumDefs() == 1) { diff --git a/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp index f069a92ac686..d652b5de6080 100644 --- a/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp @@ -55,8 +55,9 @@ extern void processInstr(MachineInstr &MI, MachineIRBuilder &MIB, static bool isMetaInstrGET(unsigned Opcode) { return Opcode == SPIRV::GET_ID || Opcode == SPIRV::GET_fID || - Opcode == SPIRV::GET_pID || Opcode == SPIRV::GET_vID || - Opcode == SPIRV::GET_vfID || Opcode == SPIRV::GET_vpID; + Opcode == SPIRV::GET_pID32 || Opcode == SPIRV::GET_pID64 || + Opcode == SPIRV::GET_vID || Opcode == SPIRV::GET_vfID || + Opcode == SPIRV::GET_vpID32 || Opcode == SPIRV::GET_vpID64; } static bool mayBeInserted(unsigned Opcode) { diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp index 2c964595fc39..ed03154d4c8d 100644 --- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp @@ -223,11 +223,12 @@ static SPIRVType *propagateSPIRVType(MachineInstr *MI, SPIRVGlobalRegistry *GR, } static std::pair -createNewIdReg(Register ValReg, unsigned Opcode, MachineRegisterInfo &MRI, +createNewIdReg(SPIRVType *SpvType, Register SrcReg, MachineRegisterInfo &MRI, const SPIRVGlobalRegistry &GR) { - LLT NewT = LLT::scalar(32); - SPIRVType *SpvType = GR.getSPIRVTypeForVReg(ValReg); + if (!SpvType) + SpvType = GR.getSPIRVTypeForVReg(SrcReg); assert(SpvType && "VReg is expected to have SPIRV type"); + LLT NewT = LLT::scalar(32); bool IsFloat = SpvType->getOpcode() == SPIRV::OpTypeFloat; bool IsVectorFloat = SpvType->getOpcode() == SPIRV::OpTypeVector && @@ -236,14 +237,38 @@ createNewIdReg(Register ValReg, unsigned Opcode, MachineRegisterInfo &MRI, IsFloat |= IsVectorFloat; auto GetIdOp = IsFloat ? SPIRV::GET_fID : SPIRV::GET_ID; auto DstClass = IsFloat ? &SPIRV::fIDRegClass : &SPIRV::IDRegClass; - if (MRI.getType(ValReg).isPointer()) { - NewT = LLT::pointer(0, 32); - GetIdOp = SPIRV::GET_pID; - DstClass = &SPIRV::pIDRegClass; - } else if (MRI.getType(ValReg).isVector()) { + if (MRI.getType(SrcReg).isPointer()) { + unsigned PtrSz = GR.getPointerSize(); + NewT = LLT::pointer(0, PtrSz); + bool IsVec = MRI.getType(SrcReg).isVector(); + if (IsVec) + NewT = LLT::fixed_vector(2, NewT); + if (PtrSz == 64) { + if (IsVec) { + GetIdOp = SPIRV::GET_vpID64; + DstClass = &SPIRV::vpID64RegClass; + } else { + GetIdOp = SPIRV::GET_pID64; + DstClass = &SPIRV::pID64RegClass; + } + } else { + if (IsVec) { + GetIdOp = SPIRV::GET_vpID32; + DstClass = &SPIRV::vpID32RegClass; + } else { + GetIdOp = SPIRV::GET_pID32; + DstClass = &SPIRV::pID32RegClass; + } + } + } else if (MRI.getType(SrcReg).isVector()) { NewT = LLT::fixed_vector(2, NewT); - GetIdOp = IsFloat ? SPIRV::GET_vfID : SPIRV::GET_vID; - DstClass = IsFloat ? &SPIRV::vfIDRegClass : &SPIRV::vIDRegClass; + if (IsFloat) { + GetIdOp = SPIRV::GET_vfID; + DstClass = &SPIRV::vfIDRegClass; + } else { + GetIdOp = SPIRV::GET_vID; + DstClass = &SPIRV::vIDRegClass; + } } Register IdReg = MRI.createGenericVirtualRegister(NewT); MRI.setRegClass(IdReg, DstClass); @@ -264,6 +289,7 @@ Register insertAssignInstr(Register Reg, Type *Ty, SPIRVType *SpirvTy, MIB.setInsertPt(*Def->getParent(), (Def->getNextNode() ? Def->getNextNode()->getIterator() : Def->getParent()->end())); + SpirvTy = SpirvTy ? SpirvTy : GR->getOrCreateSPIRVType(Ty, MIB); Register NewReg = MRI.createGenericVirtualRegister(MRI.getType(Reg)); if (auto *RC = MRI.getRegClassOrNull(Reg)) { MRI.setRegClass(NewReg, RC); @@ -271,7 +297,6 @@ Register insertAssignInstr(Register Reg, Type *Ty, SPIRVType *SpirvTy, MRI.setRegClass(NewReg, &SPIRV::IDRegClass); MRI.setRegClass(Reg, &SPIRV::IDRegClass); } - SpirvTy = SpirvTy ? SpirvTy : GR->getOrCreateSPIRVType(Ty, MIB); GR->assignSPIRVTypeToVReg(SpirvTy, Reg, MIB.getMF()); // This is to make it convenient for Legalizer to get the SPIRVType // when processing the actual MI (i.e. not pseudo one). @@ -290,11 +315,11 @@ Register insertAssignInstr(Register Reg, Type *Ty, SPIRVType *SpirvTy, void processInstr(MachineInstr &MI, MachineIRBuilder &MIB, MachineRegisterInfo &MRI, SPIRVGlobalRegistry *GR) { - unsigned Opc = MI.getOpcode(); assert(MI.getNumDefs() > 0 && MRI.hasOneUse(MI.getOperand(0).getReg())); MachineInstr &AssignTypeInst = *(MRI.use_instr_begin(MI.getOperand(0).getReg())); - auto NewReg = createNewIdReg(MI.getOperand(0).getReg(), Opc, MRI, *GR).first; + auto NewReg = + createNewIdReg(nullptr, MI.getOperand(0).getReg(), MRI, *GR).first; AssignTypeInst.getOperand(1).setReg(NewReg); MI.getOperand(0).setReg(NewReg); MIB.setInsertPt(*MI.getParent(), @@ -303,7 +328,7 @@ void processInstr(MachineInstr &MI, MachineIRBuilder &MIB, for (auto &Op : MI.operands()) { if (!Op.isReg() || Op.isDef()) continue; - auto IdOpInfo = createNewIdReg(Op.getReg(), Opc, MRI, *GR); + auto IdOpInfo = createNewIdReg(nullptr, Op.getReg(), MRI, *GR); MIB.buildInstr(IdOpInfo.second).addDef(IdOpInfo.first).addUse(Op.getReg()); Op.setReg(IdOpInfo.first); } @@ -419,6 +444,7 @@ static void processInstrsWithTypeFolding(MachineFunction &MF, processInstr(MI, MIB, MRI, GR); } } + for (MachineBasicBlock &MBB : MF) { for (MachineInstr &MI : MBB) { // We need to rewrite dst types for ASSIGN_TYPE instrs to be able @@ -431,16 +457,18 @@ static void processInstrsWithTypeFolding(MachineFunction &MF, if (!isTypeFoldingSupported(Opcode)) continue; Register DstReg = MI.getOperand(0).getReg(); - if (MRI.getType(DstReg).isVector()) + bool IsDstPtr = MRI.getType(DstReg).isPointer(); + if (IsDstPtr || MRI.getType(DstReg).isVector()) MRI.setRegClass(DstReg, &SPIRV::IDRegClass); // Don't need to reset type of register holding constant and used in - // G_ADDRSPACE_CAST, since it braaks legalizer. + // G_ADDRSPACE_CAST, since it breaks legalizer. if (Opcode == TargetOpcode::G_CONSTANT && MRI.hasOneUse(DstReg)) { MachineInstr &UseMI = *MRI.use_instr_begin(DstReg); if (UseMI.getOpcode() == TargetOpcode::G_ADDRSPACE_CAST) continue; } - MRI.setType(DstReg, LLT::scalar(32)); + MRI.setType(DstReg, IsDstPtr ? LLT::pointer(0, GR->getPointerSize()) + : LLT::scalar(32)); } } } diff --git a/llvm/lib/Target/SPIRV/SPIRVRegisterBankInfo.cpp b/llvm/lib/Target/SPIRV/SPIRVRegisterBankInfo.cpp index 5983c9229cb3..ecd99f1840d7 100644 --- a/llvm/lib/Target/SPIRV/SPIRVRegisterBankInfo.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVRegisterBankInfo.cpp @@ -27,23 +27,7 @@ using namespace llvm; const RegisterBank & SPIRVRegisterBankInfo::getRegBankFromRegClass(const TargetRegisterClass &RC, LLT Ty) const { - switch (RC.getID()) { - case SPIRV::TYPERegClassID: + if (RC.getID() == SPIRV::TYPERegClassID) return SPIRV::TYPERegBank; - case SPIRV::pIDRegClassID: - case SPIRV::IDRegClassID: - return SPIRV::IDRegBank; - case SPIRV::fIDRegClassID: - return SPIRV::fIDRegBank; - case SPIRV::vIDRegClassID: - return SPIRV::vIDRegBank; - case SPIRV::vfIDRegClassID: - return SPIRV::vfIDRegBank; - case SPIRV::vpIDRegClassID: - return SPIRV::vpIDRegBank; - case SPIRV::ANYIDRegClassID: - case SPIRV::ANYRegClassID: - return SPIRV::IDRegBank; - } - llvm_unreachable("Unknown register class"); + return SPIRV::IDRegBank; } diff --git a/llvm/lib/Target/SPIRV/SPIRVRegisterBanks.td b/llvm/lib/Target/SPIRV/SPIRVRegisterBanks.td index c7f1e172f3d4..dea2ef402d3d 100644 --- a/llvm/lib/Target/SPIRV/SPIRVRegisterBanks.td +++ b/llvm/lib/Target/SPIRV/SPIRVRegisterBanks.td @@ -8,9 +8,6 @@ // Although RegisterBankSelection is disabled we need to distinct the banks // as InstructionSelector RegClass checking code relies on them -def IDRegBank : RegisterBank<"IDBank", [ID]>; -def fIDRegBank : RegisterBank<"fIDBank", [fID]>; -def vIDRegBank : RegisterBank<"vIDBank", [vID]>; -def vfIDRegBank : RegisterBank<"vfIDBank", [vfID]>; -def vpIDRegBank : RegisterBank<"vpIDBank", [vpID]>; + def TYPERegBank : RegisterBank<"TYPEBank", [TYPE]>; +def IDRegBank : RegisterBank<"IDBank", [ID, fID, pID32, pID64, vID, vfID, vpID32, vpID64]>; diff --git a/llvm/lib/Target/SPIRV/SPIRVRegisterInfo.td b/llvm/lib/Target/SPIRV/SPIRVRegisterInfo.td index 6d2bfb91a97f..9231d22e8d83 100644 --- a/llvm/lib/Target/SPIRV/SPIRVRegisterInfo.td +++ b/llvm/lib/Target/SPIRV/SPIRVRegisterInfo.td @@ -11,39 +11,46 @@ //===----------------------------------------------------------------------===// let Namespace = "SPIRV" in { - def p0 : PtrValueType ; - - class P0Vec - : PtrValueType { - let nElem = 2; - let ElementType = p0; - let isInteger = false; - let isFP = false; - let isVector = true; + // Pointer types for patterns with the GlobalISelEmitter + def p32 : PtrValueType ; + def p64 : PtrValueType ; + + class VTPtrVec + : VTVec, ptr.Value> { + int isPointer = true; } - def v2p0 : P0Vec; - // All registers are for 32-bit identifiers, so have a single dummy register + def v2p32 : VTPtrVec<2, p32>; + def v2p64 : VTPtrVec<2, p64>; - // Class for registers that are the result of OpTypeXXX instructions + // Class for type registers def TYPE0 : Register<"TYPE0">; def TYPE : RegisterClass<"SPIRV", [i32], 32, (add TYPE0)>; - // Class for every other non-type ID + // Class for non-type registers def ID0 : Register<"ID0">; - def ID : RegisterClass<"SPIRV", [i32], 32, (add ID0)>; def fID0 : Register<"fID0">; - def fID : RegisterClass<"SPIRV", [f32], 32, (add fID0)>; - def pID0 : Register<"pID0">; - def pID : RegisterClass<"SPIRV", [p0], 32, (add pID0)>; + def pID320 : Register<"pID320">; + def pID640 : Register<"pID640">; def vID0 : Register<"vID0">; - def vID : RegisterClass<"SPIRV", [v2i32], 32, (add vID0)>; def vfID0 : Register<"vfID0">; + def vpID320 : Register<"vpID320">; + def vpID640 : Register<"vpID640">; + + def ID : RegisterClass<"SPIRV", [i32], 32, (add ID0)>; + def fID : RegisterClass<"SPIRV", [f32], 32, (add fID0)>; + def pID32 : RegisterClass<"SPIRV", [p32], 32, (add pID320)>; + def pID64 : RegisterClass<"SPIRV", [p64], 32, (add pID640)>; + def vID : RegisterClass<"SPIRV", [v2i32], 32, (add vID0)>; def vfID : RegisterClass<"SPIRV", [v2f32], 32, (add vfID0)>; - def vpID0 : Register<"vpID0">; - def vpID : RegisterClass<"SPIRV", [v2p0], 32, (add vpID0)>; + def vpID32 : RegisterClass<"SPIRV", [v2p32], 32, (add vpID320)>; + def vpID64 : RegisterClass<"SPIRV", [v2p64], 32, (add vpID640)>; - def ANYID : RegisterClass<"SPIRV", [i32, f32, p0, v2i32, v2f32], 32, (add ID, fID, pID, vID, vfID)>; + def ANYID : RegisterClass< + "SPIRV", + [i32, f32, p32, p64, v2i32, v2f32, v2p32, v2p64], + 32, + (add ID0, fID0, pID320, pID640, vID0, vfID0, vpID320, vpID640)>; // A few instructions like OpName can take ids from both type and non-type // instructions, so we need a super-class to allow for both to count as valid diff --git a/llvm/test/CodeGen/SPIRV/instructions/select-phi.ll b/llvm/test/CodeGen/SPIRV/instructions/select-phi.ll index afc75c616f02..3828fe89e60a 100644 --- a/llvm/test/CodeGen/SPIRV/instructions/select-phi.ll +++ b/llvm/test/CodeGen/SPIRV/instructions/select-phi.ll @@ -1,6 +1,10 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s +; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s + ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown --translator-compatibility-mode %s -o - -filetype=obj | spirv-val %} ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown --translator-compatibility-mode %s -o - -filetype=obj | spirv-val %} +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-DAG: %[[Char:.*]] = OpTypeInt 8 0 ; CHECK-DAG: %[[Long:.*]] = OpTypeInt 32 0 diff --git a/llvm/test/CodeGen/SPIRV/instructions/select-ptr-load.ll b/llvm/test/CodeGen/SPIRV/instructions/select-ptr-load.ll new file mode 100644 index 000000000000..0ff28952f808 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/instructions/select-ptr-load.ll @@ -0,0 +1,25 @@ +; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +; CHECK-SPIRV-DAG: %[[Float:.*]] = OpTypeFloat 32 +; CHECK-SPIRV-DAG: %[[FloatPtr:.*]] = OpTypePointer Function %[[Float]] +; CHECK-SPIRV: OpInBoundsPtrAccessChain %[[FloatPtr]] +; CHECK-SPIRV: OpInBoundsPtrAccessChain %[[FloatPtr]] +; CHECK-SPIRV: OpSelect %[[FloatPtr]] +; CHECK-SPIRV: OpLoad %[[Float]] + +%struct = type { [3 x float] } + +define spir_kernel void @bar(i1 %sw) { +entry: + %var1 = alloca %struct + %var2 = alloca %struct + %elem1 = getelementptr inbounds [3 x float], ptr %var1, i64 0, i64 0 + %elem2 = getelementptr inbounds [3 x float], ptr %var2, i64 0, i64 1 + %elem = select i1 %sw, ptr %elem1, ptr %elem2 + %res = load float, ptr %elem + ret void +} diff --git a/llvm/test/CodeGen/SPIRV/instructions/select.ll b/llvm/test/CodeGen/SPIRV/instructions/select.ll index c4176b17abb4..9234b97157d9 100644 --- a/llvm/test/CodeGen/SPIRV/instructions/select.ll +++ b/llvm/test/CodeGen/SPIRV/instructions/select.ll @@ -1,6 +1,9 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} +; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} + ; CHECK-DAG: OpName [[SCALARi32:%.+]] "select_i32" ; CHECK-DAG: OpName [[SCALARPTR:%.+]] "select_ptr" ; CHECK-DAG: OpName [[VEC2i32:%.+]] "select_i32v2" diff --git a/llvm/test/CodeGen/SPIRV/select.ll b/llvm/test/CodeGen/SPIRV/select-builtin.ll similarity index 67% rename from llvm/test/CodeGen/SPIRV/select.ll rename to llvm/test/CodeGen/SPIRV/select-builtin.ll index b34e91be1dbc..6717970d160f 100644 --- a/llvm/test/CodeGen/SPIRV/select.ll +++ b/llvm/test/CodeGen/SPIRV/select-builtin.ll @@ -1,4 +1,6 @@ ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-SPIRV: OpSelect -- GitLab From 42d801d4e42ff8c47c3a24d562774851e3a424f5 Mon Sep 17 00:00:00 2001 From: Vyacheslav Levytskyy Date: Wed, 17 Apr 2024 11:50:55 +0200 Subject: [PATCH 025/862] [SPIR-V] Account for zext in a llvm intrinsic call (#88903) This PR addresses an issue that may arise when an integer argument size differs from a machine word size for the target in a call to llvm intrinsic. The following example demonstrates the issue: ``` @__const.test.arr = private unnamed_addr addrspace(2) constant [3 x i32] [i32 1, i32 2, i32 3] define spir_func void @test() { entry: %arr = alloca [3 x i32], align 4 %dest = bitcast ptr %arr to ptr call void @llvm.memcpy.p0.p2.i32(ptr align 4 %dest, ptr addrspace(2) align 4 @__const.test.arr, i32 1024, i1 false) ret void } declare void @llvm.memcpy.p0.p2.i32(ptr nocapture writeonly, ptr addrspace(2) nocapture readonly, i32, i1) ``` Depending on the target this code may work or may fail without this PR due to the fact that IR Translation step introduces additional `zext` when type of the 3rd argument of `@llvm.memcpy.p0.p2.i32` differs from machine word. This PR addresses the issue by adding type deduction for a newly inserted G_ZEXT generic opcode. --- llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp | 25 +++++++++ .../CodeGen/SPIRV/transcoding/memcpy-zext.ll | 43 ++++++++++++++++ .../spirv-private-array-initialization.ll | 51 +++++++++++-------- 3 files changed, 99 insertions(+), 20 deletions(-) create mode 100644 llvm/test/CodeGen/SPIRV/transcoding/memcpy-zext.ll diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp index ed03154d4c8d..d16f6d5bf67e 100644 --- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp @@ -171,6 +171,12 @@ static void insertBitcasts(MachineFunction &MF, SPIRVGlobalRegistry *GR, // %1 = G_GLOBAL_VALUE // %2 = COPY %1 // %3 = G_ADDRSPACE_CAST %2 +// +// or +// +// %1 = G_ZEXT %2 +// G_MEMCPY ... %2 ... +// // New registers have no SPIRVType and no register class info. // // Set SPIRVType for GV, propagate it from GV to other instructions, @@ -200,6 +206,24 @@ static SPIRVType *propagateSPIRVType(MachineInstr *MI, SPIRVGlobalRegistry *GR, SpirvTy = GR->getOrCreateSPIRVType(Ty, MIB); break; } + case TargetOpcode::G_ZEXT: { + if (MI->getOperand(1).isReg()) { + if (MachineInstr *DefInstr = + MRI.getVRegDef(MI->getOperand(1).getReg())) { + if (SPIRVType *Def = propagateSPIRVType(DefInstr, GR, MRI, MIB)) { + unsigned CurrentBW = GR->getScalarOrVectorBitWidth(Def); + unsigned ExpectedBW = + std::max(MRI.getType(Reg).getScalarSizeInBits(), CurrentBW); + unsigned NumElements = GR->getScalarOrVectorComponentCount(Def); + SpirvTy = GR->getOrCreateSPIRVIntegerType(ExpectedBW, MIB); + if (NumElements > 1) + SpirvTy = + GR->getOrCreateSPIRVVectorType(SpirvTy, NumElements, MIB); + } + } + } + break; + } case TargetOpcode::G_TRUNC: case TargetOpcode::G_ADDRSPACE_CAST: case TargetOpcode::G_PTR_ADD: @@ -415,6 +439,7 @@ static void generateAssignInstrs(MachineFunction &MF, SPIRVGlobalRegistry *GR, } insertAssignInstr(Reg, Ty, nullptr, GR, MIB, MRI); } else if (MI.getOpcode() == TargetOpcode::G_TRUNC || + MI.getOpcode() == TargetOpcode::G_ZEXT || MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE || MI.getOpcode() == TargetOpcode::COPY || MI.getOpcode() == TargetOpcode::G_ADDRSPACE_CAST) { diff --git a/llvm/test/CodeGen/SPIRV/transcoding/memcpy-zext.ll b/llvm/test/CodeGen/SPIRV/transcoding/memcpy-zext.ll new file mode 100644 index 000000000000..ea0197548a81 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/transcoding/memcpy-zext.ll @@ -0,0 +1,43 @@ +; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-32 +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} +; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-64 +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +; CHECK-64-DAG: %[[#i64:]] = OpTypeInt 64 0 + +; CHECK-DAG: %[[#i8:]] = OpTypeInt 8 0 +; CHECK-DAG: %[[#i32:]] = OpTypeInt 32 0 +; CHECK-DAG: %[[#one:]] = OpConstant %[[#i32]] 1 +; CHECK-DAG: %[[#two:]] = OpConstant %[[#i32]] 2 +; CHECK-DAG: %[[#three:]] = OpConstant %[[#i32]] 3 +; CHECK-DAG: %[[#i32x3:]] = OpTypeArray %[[#i32]] %[[#three]] +; CHECK-DAG: %[[#test_arr_init:]] = OpConstantComposite %[[#i32x3]] %[[#one]] %[[#two]] %[[#three]] +; CHECK-DAG: %[[#szconst1024:]] = OpConstant %[[#i32]] 1024 +; CHECK-DAG: %[[#szconst42:]] = OpConstant %[[#i8]] 42 +; CHECK-DAG: %[[#const_i32x3_ptr:]] = OpTypePointer UniformConstant %[[#i32x3]] +; CHECK-DAG: %[[#test_arr:]] = OpVariable %[[#const_i32x3_ptr]] UniformConstant %[[#test_arr_init]] +; CHECK-DAG: %[[#i32x3_ptr:]] = OpTypePointer Function %[[#i32x3]] +; CHECK: %[[#arr:]] = OpVariable %[[#i32x3_ptr]] Function + +; CHECK-32: OpCopyMemorySized %[[#arr]] %[[#test_arr]] %[[#szconst1024]] +; CHECK-64: %[[#szconstext1024:]] = OpUConvert %[[#i64:]] %[[#szconst1024:]] +; CHECK-64: OpCopyMemorySized %[[#arr]] %[[#test_arr]] %[[#szconstext1024]] + +; CHECK-32: %[[#szconstext42:]] = OpUConvert %[[#i32:]] %[[#szconst42:]] +; CHECK-32: OpCopyMemorySized %[[#arr]] %[[#test_arr]] %[[#szconstext42]] +; CHECK-64: %[[#szconstext42:]] = OpUConvert %[[#i64:]] %[[#szconst42:]] +; CHECK-64: OpCopyMemorySized %[[#arr]] %[[#test_arr]] %[[#szconstext42]] + +@__const.test.arr = private unnamed_addr addrspace(2) constant [3 x i32] [i32 1, i32 2, i32 3] + +define spir_func void @test() { +entry: + %arr = alloca [3 x i32], align 4 + %dest = bitcast ptr %arr to ptr + call void @llvm.memcpy.p0.p2.i32(ptr align 4 %dest, ptr addrspace(2) align 4 @__const.test.arr, i32 1024, i1 false) + call void @llvm.memcpy.p0.p2.i8(ptr align 4 %dest, ptr addrspace(2) align 4 @__const.test.arr, i8 42, i1 false) + ret void +} + +declare void @llvm.memcpy.p0.p2.i32(ptr nocapture writeonly, ptr addrspace(2) nocapture readonly, i32, i1) +declare void @llvm.memcpy.p0.p2.i8(ptr nocapture writeonly, ptr addrspace(2) nocapture readonly, i8, i1) diff --git a/llvm/test/CodeGen/SPIRV/transcoding/spirv-private-array-initialization.ll b/llvm/test/CodeGen/SPIRV/transcoding/spirv-private-array-initialization.ll index e0172ec3c1bd..04fb39118034 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/spirv-private-array-initialization.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/spirv-private-array-initialization.ll @@ -1,23 +1,34 @@ -; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV -; -; CHECK-SPIRV-DAG: %[[#i32:]] = OpTypeInt 32 0 -; CHECK-SPIRV-DAG: %[[#one:]] = OpConstant %[[#i32]] 1 -; CHECK-SPIRV-DAG: %[[#two:]] = OpConstant %[[#i32]] 2 -; CHECK-SPIRV-DAG: %[[#three:]] = OpConstant %[[#i32]] 3 -; CHECK-SPIRV-DAG: %[[#i32x3:]] = OpTypeArray %[[#i32]] %[[#three]] -; CHECK-SPIRV-DAG: %[[#test_arr_init:]] = OpConstantComposite %[[#i32x3]] %[[#one]] %[[#two]] %[[#three]] -; CHECK-SPIRV-DAG: %[[#twelve:]] = OpConstant %[[#i32]] 12 -; CHECK-SPIRV-DAG: %[[#const_i32x3_ptr:]] = OpTypePointer UniformConstant %[[#i32x3]] - -; CHECK-SPIRV: %[[#test_arr2:]] = OpVariable %[[#const_i32x3_ptr]] UniformConstant %[[#test_arr_init]] -; CHECK-SPIRV: %[[#test_arr:]] = OpVariable %[[#const_i32x3_ptr]] UniformConstant %[[#test_arr_init]] - -; CHECK-SPIRV-DAG: %[[#i32x3_ptr:]] = OpTypePointer Function %[[#i32x3]] - -; CHECK-SPIRV: %[[#arr:]] = OpVariable %[[#i32x3_ptr]] Function -; CHECK-SPIRV: %[[#arr2:]] = OpVariable %[[#i32x3_ptr]] Function -; CHECK-SPIRV: OpCopyMemorySized %[[#arr]] %[[#test_arr]] %[[#twelve]] Aligned 4 -; CHECK-SPIRV: OpCopyMemorySized %[[#arr2]] %[[#test_arr2]] %[[#twelve]] Aligned 4 +; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK-SPIRV,CHECK-SPIRV-32 +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} +; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK-SPIRV,CHECK-SPIRV-64 +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +; CHECK-SPIRV-64-DAG: %[[#i64:]] = OpTypeInt 64 0 + +; CHECK-SPIRV-DAG: %[[#i32:]] = OpTypeInt 32 0 +; CHECK-SPIRV-DAG: %[[#one:]] = OpConstant %[[#i32]] 1 +; CHECK-SPIRV-DAG: %[[#two:]] = OpConstant %[[#i32]] 2 +; CHECK-SPIRV-DAG: %[[#three:]] = OpConstant %[[#i32]] 3 +; CHECK-SPIRV-DAG: %[[#i32x3:]] = OpTypeArray %[[#i32]] %[[#three]] +; CHECK-SPIRV-DAG: %[[#test_arr_init:]] = OpConstantComposite %[[#i32x3]] %[[#one]] %[[#two]] %[[#three]] +; CHECK-SPIRV-DAG: %[[#twelve:]] = OpConstant %[[#i32]] 12 +; CHECK-SPIRV-DAG: %[[#const_i32x3_ptr:]] = OpTypePointer UniformConstant %[[#i32x3]] + +; CHECK-SPIRV: %[[#test_arr2:]] = OpVariable %[[#const_i32x3_ptr]] UniformConstant %[[#test_arr_init]] +; CHECK-SPIRV: %[[#test_arr:]] = OpVariable %[[#const_i32x3_ptr]] UniformConstant %[[#test_arr_init]] + +; CHECK-SPIRV-DAG: %[[#i32x3_ptr:]] = OpTypePointer Function %[[#i32x3]] + +; CHECK-SPIRV: %[[#arr:]] = OpVariable %[[#i32x3_ptr]] Function +; CHECK-SPIRV: %[[#arr2:]] = OpVariable %[[#i32x3_ptr]] Function + +; CHECK-SPIRV-32: OpCopyMemorySized %[[#arr]] %[[#test_arr]] %[[#twelve]] Aligned 4 +; CHECK-SPIRV-32: OpCopyMemorySized %[[#arr2]] %[[#test_arr2]] %[[#twelve]] Aligned 4 + +; CHECK-SPIRV-64: %[[#twelvezext1:]] = OpUConvert %[[#i64:]] %[[#twelve:]] +; CHECK-SPIRV-64: OpCopyMemorySized %[[#arr]] %[[#test_arr]] %[[#twelvezext1]] Aligned 4 +; CHECK-SPIRV-64: %[[#twelvezext2:]] = OpUConvert %[[#i64:]] %[[#twelve:]] +; CHECK-SPIRV-64: OpCopyMemorySized %[[#arr2]] %[[#test_arr2]] %[[#twelvezext2]] Aligned 4 @__const.test.arr = private unnamed_addr addrspace(2) constant [3 x i32] [i32 1, i32 2, i32 3], align 4 -- GitLab From fa61f062a515be92a98cac64a9193498918c1225 Mon Sep 17 00:00:00 2001 From: harishch4 Date: Wed, 17 Apr 2024 15:21:45 +0530 Subject: [PATCH 026/862] Fix threadprivate variable scope inside BLOCK construct. (#88921) When a local variable inside a BLOCK construct is used as threadprivate variable, llvm-flang throws below error: > error: The THREADPRIVATE directive and the common block or variable in it must appear in the same declaration section of a scoping unit --- flang/lib/Semantics/check-omp-structure.cpp | 2 +- flang/test/Lower/OpenMP/threadprivate-hlfir.f90 | 1 + flang/test/Semantics/OpenMP/threadprivate07.f90 | 15 +++++++++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 flang/test/Semantics/OpenMP/threadprivate07.f90 diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp index e85d8d1f7ab5..bafa242a7930 100644 --- a/flang/lib/Semantics/check-omp-structure.cpp +++ b/flang/lib/Semantics/check-omp-structure.cpp @@ -1048,7 +1048,7 @@ void OmpStructureChecker::CheckThreadprivateOrDeclareTargetVar( name->symbol->GetUltimate().owner(); if (!curScope.IsTopLevel()) { const semantics::Scope &declScope = - GetProgramUnitContaining(curScope); + GetProgramUnitOrBlockConstructContaining(curScope); const semantics::Symbol *sym{ declScope.parent().FindSymbol(name->symbol->name())}; if (sym && diff --git a/flang/test/Lower/OpenMP/threadprivate-hlfir.f90 b/flang/test/Lower/OpenMP/threadprivate-hlfir.f90 index d39ae1e70118..7d02987c5ead 100644 --- a/flang/test/Lower/OpenMP/threadprivate-hlfir.f90 +++ b/flang/test/Lower/OpenMP/threadprivate-hlfir.f90 @@ -24,3 +24,4 @@ subroutine sub() print *, a !$omp end parallel end subroutine + diff --git a/flang/test/Semantics/OpenMP/threadprivate07.f90 b/flang/test/Semantics/OpenMP/threadprivate07.f90 new file mode 100644 index 000000000000..c9a006ca0e08 --- /dev/null +++ b/flang/test/Semantics/OpenMP/threadprivate07.f90 @@ -0,0 +1,15 @@ +! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp + +! Check Threadprivate Directive with local variable of a BLOCK construct. + +program main + call sub1() + print *, 'pass' +end program main + +subroutine sub1() + BLOCK + integer, save :: a + !$omp threadprivate(a) + END BLOCK +end subroutine -- GitLab From cbe148b730a04fc95eda9a43903f0af487884a96 Mon Sep 17 00:00:00 2001 From: Mel Chen Date: Wed, 17 Apr 2024 17:59:52 +0800 Subject: [PATCH 027/862] [LV][NFC] Remove the declaration of function `fixReduction`. (#88491) --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 44885a95bd10..20059f9d62d5 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -616,9 +616,6 @@ protected: void fixFixedOrderRecurrence(VPFirstOrderRecurrencePHIRecipe *PhiR, VPTransformState &State); - /// Create code for the loop exit value of the reduction. - void fixReduction(VPReductionPHIRecipe *Phi, VPTransformState &State); - /// Iteratively sink the scalarized operands of a predicated instruction into /// the block that was created for it. void sinkScalarOperands(Instruction *PredInst); -- GitLab From a9bafe91dd088c5fa6f074c14dd3a1af25f00457 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Wed, 17 Apr 2024 11:00:58 +0100 Subject: [PATCH 028/862] [VPlan] Split VPWidenMemoryInstructionRecipe (NFCI). (#87411) This patch introduces a new VPWidenMemoryRecipe base class and distinct sub-classes to model loads and stores. This is a first step in an effort to simplify and modularize code generation for widened loads and stores and enable adding further more specialized memory recipes. PR: https://github.com/llvm/llvm-project/pull/87411 --- .../Transforms/Vectorize/LoopVectorize.cpp | 191 +++++++++--------- .../Transforms/Vectorize/VPRecipeBuilder.h | 6 +- llvm/lib/Transforms/Vectorize/VPlan.h | 161 +++++++++------ .../Transforms/Vectorize/VPlanAnalysis.cpp | 9 +- llvm/lib/Transforms/Vectorize/VPlanAnalysis.h | 4 +- .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 44 ++-- .../Transforms/Vectorize/VPlanTransforms.cpp | 23 +-- llvm/lib/Transforms/Vectorize/VPlanValue.h | 5 +- .../Transforms/Vectorize/VPlanVerifier.cpp | 2 +- .../AArch64/vector-reverse-mask4.ll | 2 +- .../LoopVectorize/X86/masked_load_store.ll | 12 +- .../Transforms/Vectorize/VPlanHCFGTest.cpp | 4 +- .../Transforms/Vectorize/VPlanTest.cpp | 9 +- 13 files changed, 249 insertions(+), 223 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 20059f9d62d5..a8272f450253 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -545,11 +545,6 @@ public: // Return true if any runtime check is added. bool areSafetyChecksAdded() { return AddedSafetyChecks; } - /// A type for vectorized values in the new loop. Each value from the - /// original loop, when vectorized, is represented by UF vector values in the - /// new unrolled loop, where UF is the unroll factor. - using VectorParts = SmallVector; - /// A helper function to scalarize a single Instruction in the innermost loop. /// Generates a sequence of scalar instances for each lane between \p MinLane /// and \p MaxLane, times each part between \p MinPart and \p MaxPart, @@ -8086,7 +8081,7 @@ void VPRecipeBuilder::createBlockInMask(BasicBlock *BB) { BlockMaskCache[BB] = BlockMask; } -VPWidenMemoryInstructionRecipe * +VPWidenMemoryRecipe * VPRecipeBuilder::tryToWidenMemory(Instruction *I, ArrayRef Operands, VFRange &Range) { assert((isa(I) || isa(I)) && @@ -8131,12 +8126,12 @@ VPRecipeBuilder::tryToWidenMemory(Instruction *I, ArrayRef Operands, Ptr = VectorPtr; } if (LoadInst *Load = dyn_cast(I)) - return new VPWidenMemoryInstructionRecipe(*Load, Ptr, Mask, Consecutive, - Reverse, I->getDebugLoc()); + return new VPWidenLoadRecipe(*Load, Ptr, Mask, Consecutive, Reverse, + I->getDebugLoc()); StoreInst *Store = cast(I); - return new VPWidenMemoryInstructionRecipe( - *Store, Ptr, Operands[0], Mask, Consecutive, Reverse, I->getDebugLoc()); + return new VPWidenStoreRecipe(*Store, Ptr, Operands[0], Mask, Consecutive, + Reverse, I->getDebugLoc()); } /// Creates a VPWidenIntOrFpInductionRecpipe for \p Phi. If needed, it will also @@ -8775,13 +8770,12 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { // for this VPlan, replace the Recipes widening its memory instructions with a // single VPInterleaveRecipe at its insertion point. for (const auto *IG : InterleaveGroups) { - auto *Recipe = cast( - RecipeBuilder.getRecipe(IG->getInsertPos())); + auto *Recipe = + cast(RecipeBuilder.getRecipe(IG->getInsertPos())); SmallVector StoredValues; for (unsigned i = 0; i < IG->getFactor(); ++i) if (auto *SI = dyn_cast_or_null(IG->getMember(i))) { - auto *StoreR = - cast(RecipeBuilder.getRecipe(SI)); + auto *StoreR = cast(RecipeBuilder.getRecipe(SI)); StoredValues.push_back(StoreR->getStoredValue()); } @@ -9368,92 +9362,27 @@ static Instruction *lowerLoadUsingVectorIntrinsics(IRBuilderBase &Builder, return Call; } -void VPWidenMemoryInstructionRecipe::execute(VPTransformState &State) { - VPValue *StoredValue = isStore() ? getStoredValue() : nullptr; - - // Attempt to issue a wide load. - LoadInst *LI = dyn_cast(&Ingredient); - StoreInst *SI = dyn_cast(&Ingredient); - - assert((LI || SI) && "Invalid Load/Store instruction"); - assert((!SI || StoredValue) && "No stored value provided for widened store"); - assert((!LI || !StoredValue) && "Stored value provided for widened load"); +void VPWidenLoadRecipe::execute(VPTransformState &State) { + auto *LI = cast(&Ingredient); Type *ScalarDataTy = getLoadStoreType(&Ingredient); - auto *DataTy = VectorType::get(ScalarDataTy, State.VF); const Align Alignment = getLoadStoreAlignment(&Ingredient); - bool CreateGatherScatter = !isConsecutive(); + bool CreateGather = !isConsecutive(); auto &Builder = State.Builder; - InnerLoopVectorizer::VectorParts BlockInMaskParts(State.UF); - bool isMaskRequired = getMask(); - if (isMaskRequired) { - // Mask reversal is only needed for non-all-one (null) masks, as reverse of - // a null all-one mask is a null mask. - for (unsigned Part = 0; Part < State.UF; ++Part) { - Value *Mask = State.get(getMask(), Part); + State.setDebugLocFrom(getDebugLoc()); + for (unsigned Part = 0; Part < State.UF; ++Part) { + Value *NewLI; + Value *Mask = nullptr; + if (auto *VPMask = getMask()) { + // Mask reversal is only needed for non-all-one (null) masks, as reverse + // of a null all-one mask is a null mask. + Mask = State.get(VPMask, Part); if (isReverse()) Mask = Builder.CreateVectorReverse(Mask, "reverse"); - BlockInMaskParts[Part] = Mask; - } - } - - // Handle Stores: - if (SI) { - State.setDebugLocFrom(getDebugLoc()); - - for (unsigned Part = 0; Part < State.UF; ++Part) { - Instruction *NewSI = nullptr; - Value *StoredVal = State.get(StoredValue, Part); - // TODO: split this into several classes for better design. - if (State.EVL) { - assert(State.UF == 1 && "Expected only UF == 1 when vectorizing with " - "explicit vector length."); - assert(cast(State.EVL)->getOpcode() == - VPInstruction::ExplicitVectorLength && - "EVL must be VPInstruction::ExplicitVectorLength."); - Value *EVL = State.get(State.EVL, VPIteration(0, 0)); - // If EVL is not nullptr, then EVL must be a valid value set during plan - // creation, possibly default value = whole vector register length. EVL - // is created only if TTI prefers predicated vectorization, thus if EVL - // is not nullptr it also implies preference for predicated - // vectorization. - // FIXME: Support reverse store after vp_reverse is added. - Value *MaskPart = isMaskRequired ? BlockInMaskParts[Part] : nullptr; - NewSI = lowerStoreUsingVectorIntrinsics( - Builder, State.get(getAddr(), Part, !CreateGatherScatter), - StoredVal, CreateGatherScatter, MaskPart, EVL, Alignment); - } else if (CreateGatherScatter) { - Value *MaskPart = isMaskRequired ? BlockInMaskParts[Part] : nullptr; - Value *VectorGep = State.get(getAddr(), Part); - NewSI = Builder.CreateMaskedScatter(StoredVal, VectorGep, Alignment, - MaskPart); - } else { - if (isReverse()) { - // If we store to reverse consecutive memory locations, then we need - // to reverse the order of elements in the stored value. - StoredVal = Builder.CreateVectorReverse(StoredVal, "reverse"); - // We don't want to update the value in the map as it might be used in - // another expression. So don't call resetVectorValue(StoredVal). - } - auto *VecPtr = State.get(getAddr(), Part, /*IsScalar*/ true); - if (isMaskRequired) - NewSI = Builder.CreateMaskedStore(StoredVal, VecPtr, Alignment, - BlockInMaskParts[Part]); - else - NewSI = Builder.CreateAlignedStore(StoredVal, VecPtr, Alignment); - } - State.addMetadata(NewSI, SI); } - return; - } - // Handle loads. - assert(LI && "Must have a load instruction"); - State.setDebugLocFrom(getDebugLoc()); - for (unsigned Part = 0; Part < State.UF; ++Part) { - Value *NewLI; // TODO: split this into several classes for better design. if (State.EVL) { assert(State.UF == 1 && "Expected only UF == 1 when vectorizing with " @@ -9468,22 +9397,20 @@ void VPWidenMemoryInstructionRecipe::execute(VPTransformState &State) { // is not nullptr it also implies preference for predicated // vectorization. // FIXME: Support reverse loading after vp_reverse is added. - Value *MaskPart = isMaskRequired ? BlockInMaskParts[Part] : nullptr; NewLI = lowerLoadUsingVectorIntrinsics( - Builder, DataTy, State.get(getAddr(), Part, !CreateGatherScatter), - CreateGatherScatter, MaskPart, EVL, Alignment); - } else if (CreateGatherScatter) { - Value *MaskPart = isMaskRequired ? BlockInMaskParts[Part] : nullptr; + Builder, DataTy, State.get(getAddr(), Part, !CreateGather), + CreateGather, Mask, EVL, Alignment); + } else if (CreateGather) { Value *VectorGep = State.get(getAddr(), Part); - NewLI = Builder.CreateMaskedGather(DataTy, VectorGep, Alignment, MaskPart, + NewLI = Builder.CreateMaskedGather(DataTy, VectorGep, Alignment, Mask, nullptr, "wide.masked.gather"); State.addMetadata(NewLI, LI); } else { auto *VecPtr = State.get(getAddr(), Part, /*IsScalar*/ true); - if (isMaskRequired) - NewLI = Builder.CreateMaskedLoad( - DataTy, VecPtr, Alignment, BlockInMaskParts[Part], - PoisonValue::get(DataTy), "wide.masked.load"); + if (Mask) + NewLI = Builder.CreateMaskedLoad(DataTy, VecPtr, Alignment, Mask, + PoisonValue::get(DataTy), + "wide.masked.load"); else NewLI = Builder.CreateAlignedLoad(DataTy, VecPtr, Alignment, "wide.load"); @@ -9494,7 +9421,69 @@ void VPWidenMemoryInstructionRecipe::execute(VPTransformState &State) { NewLI = Builder.CreateVectorReverse(NewLI, "reverse"); } - State.set(getVPSingleValue(), NewLI, Part); + State.set(this, NewLI, Part); + } +} + +void VPWidenStoreRecipe::execute(VPTransformState &State) { + auto *SI = cast(&Ingredient); + + VPValue *StoredVPValue = getStoredValue(); + bool CreateScatter = !isConsecutive(); + const Align Alignment = getLoadStoreAlignment(&Ingredient); + + auto &Builder = State.Builder; + State.setDebugLocFrom(getDebugLoc()); + + for (unsigned Part = 0; Part < State.UF; ++Part) { + Instruction *NewSI = nullptr; + Value *Mask = nullptr; + if (auto *VPMask = getMask()) { + // Mask reversal is only needed for non-all-one (null) masks, as reverse + // of a null all-one mask is a null mask. + Mask = State.get(VPMask, Part); + if (isReverse()) + Mask = Builder.CreateVectorReverse(Mask, "reverse"); + } + + Value *StoredVal = State.get(StoredVPValue, Part); + if (isReverse()) { + assert(!State.EVL && "reversing not yet implemented with EVL"); + // If we store to reverse consecutive memory locations, then we need + // to reverse the order of elements in the stored value. + StoredVal = Builder.CreateVectorReverse(StoredVal, "reverse"); + // We don't want to update the value in the map as it might be used in + // another expression. So don't call resetVectorValue(StoredVal). + } + // TODO: split this into several classes for better design. + if (State.EVL) { + assert(State.UF == 1 && "Expected only UF == 1 when vectorizing with " + "explicit vector length."); + assert(cast(State.EVL)->getOpcode() == + VPInstruction::ExplicitVectorLength && + "EVL must be VPInstruction::ExplicitVectorLength."); + Value *EVL = State.get(State.EVL, VPIteration(0, 0)); + // If EVL is not nullptr, then EVL must be a valid value set during plan + // creation, possibly default value = whole vector register length. EVL + // is created only if TTI prefers predicated vectorization, thus if EVL + // is not nullptr it also implies preference for predicated + // vectorization. + // FIXME: Support reverse store after vp_reverse is added. + NewSI = lowerStoreUsingVectorIntrinsics( + Builder, State.get(getAddr(), Part, !CreateScatter), StoredVal, + CreateScatter, Mask, EVL, Alignment); + } else if (CreateScatter) { + Value *VectorGep = State.get(getAddr(), Part); + NewSI = + Builder.CreateMaskedScatter(StoredVal, VectorGep, Alignment, Mask); + } else { + auto *VecPtr = State.get(getAddr(), Part, /*IsScalar*/ true); + if (Mask) + NewSI = Builder.CreateMaskedStore(StoredVal, VecPtr, Alignment, Mask); + else + NewSI = Builder.CreateAlignedStore(StoredVal, VecPtr, Alignment); + } + State.addMetadata(NewSI, SI); } } diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h index 605b47fa0a46..b4c7ab02f928 100644 --- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h +++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h @@ -69,9 +69,9 @@ class VPRecipeBuilder { /// Check if the load or store instruction \p I should widened for \p /// Range.Start and potentially masked. Such instructions are handled by a /// recipe that takes an additional VPInstruction for the mask. - VPWidenMemoryInstructionRecipe *tryToWidenMemory(Instruction *I, - ArrayRef Operands, - VFRange &Range); + VPWidenMemoryRecipe *tryToWidenMemory(Instruction *I, + ArrayRef Operands, + VFRange &Range); /// Check if an induction recipe should be constructed for \p Phi. If so build /// and return it. If not, return null. diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index d86a81d4fb4c..148227f1f1a5 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -875,7 +875,8 @@ public: return true; case VPRecipeBase::VPInterleaveSC: case VPRecipeBase::VPBranchOnMaskSC: - case VPRecipeBase::VPWidenMemoryInstructionSC: + case VPRecipeBase::VPWidenLoadSC: + case VPRecipeBase::VPWidenStoreSC: // TODO: Widened stores don't define a value, but widened loads do. Split // the recipes to be able to make widened loads VPSingleDefRecipes. return false; @@ -2280,68 +2281,62 @@ public: } }; -/// A Recipe for widening load/store operations. -/// The recipe uses the following VPValues: -/// - For load: Address, optional mask -/// - For store: Address, stored value, optional mask -/// TODO: We currently execute only per-part unless a specific instance is -/// provided. -class VPWidenMemoryInstructionRecipe : public VPRecipeBase { +/// A common base class for widening memory operations. An optional mask can be +/// provided as the last operand. +class VPWidenMemoryRecipe : public VPRecipeBase { +protected: Instruction &Ingredient; - // Whether the loaded-from / stored-to addresses are consecutive. + /// Whether the accessed addresses are consecutive. bool Consecutive; - // Whether the consecutive loaded/stored addresses are in reverse order. + /// Whether the consecutive accessed addresses are in reverse order. bool Reverse; + /// Whether the memory access is masked. + bool IsMasked = false; + void setMask(VPValue *Mask) { + assert(!IsMasked && "cannot re-set mask"); if (!Mask) return; addOperand(Mask); + IsMasked = true; } - bool isMasked() const { - return isStore() ? getNumOperands() == 3 : getNumOperands() == 2; + VPWidenMemoryRecipe(const char unsigned SC, Instruction &I, + std::initializer_list Operands, + bool Consecutive, bool Reverse, DebugLoc DL) + : VPRecipeBase(SC, Operands, DL), Ingredient(I), Consecutive(Consecutive), + Reverse(Reverse) { + assert((Consecutive || !Reverse) && "Reverse implies consecutive"); } public: - VPWidenMemoryInstructionRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask, - bool Consecutive, bool Reverse, DebugLoc DL) - : VPRecipeBase(VPDef::VPWidenMemoryInstructionSC, {Addr}, DL), - Ingredient(Load), Consecutive(Consecutive), Reverse(Reverse) { - assert((Consecutive || !Reverse) && "Reverse implies consecutive"); - new VPValue(this, &Load); - setMask(Mask); - } + VPWidenMemoryRecipe *clone() override = 0; - VPWidenMemoryInstructionRecipe(StoreInst &Store, VPValue *Addr, - VPValue *StoredValue, VPValue *Mask, - bool Consecutive, bool Reverse, DebugLoc DL) - : VPRecipeBase(VPDef::VPWidenMemoryInstructionSC, {Addr, StoredValue}, - DL), - Ingredient(Store), Consecutive(Consecutive), Reverse(Reverse) { - assert((Consecutive || !Reverse) && "Reverse implies consecutive"); - setMask(Mask); + static inline bool classof(const VPRecipeBase *R) { + return R->getVPDefID() == VPDef::VPWidenLoadSC || + R->getVPDefID() == VPDef::VPWidenStoreSC; } - VPWidenMemoryInstructionRecipe *clone() override { - if (isStore()) - return new VPWidenMemoryInstructionRecipe( - cast(Ingredient), getAddr(), getStoredValue(), getMask(), - Consecutive, Reverse, getDebugLoc()); - - return new VPWidenMemoryInstructionRecipe(cast(Ingredient), - getAddr(), getMask(), Consecutive, - Reverse, getDebugLoc()); + static inline bool classof(const VPUser *U) { + auto *R = dyn_cast(U); + return R && classof(R); } - VP_CLASSOF_IMPL(VPDef::VPWidenMemoryInstructionSC) + /// Return whether the loaded-from / stored-to addresses are consecutive. + bool isConsecutive() const { return Consecutive; } + + /// Return whether the consecutive loaded/stored addresses are in reverse + /// order. + bool isReverse() const { return Reverse; } /// Return the address accessed by this recipe. - VPValue *getAddr() const { - return getOperand(0); // Address is the 1st, mandatory operand. - } + VPValue *getAddr() const { return getOperand(0); } + + /// Returns true if the recipe is masked. + bool isMasked() const { return IsMasked; } /// Return the mask used by this recipe. Note that a full mask is represented /// by a nullptr. @@ -2350,23 +2345,34 @@ public: return isMasked() ? getOperand(getNumOperands() - 1) : nullptr; } - /// Returns true if this recipe is a store. - bool isStore() const { return isa(Ingredient); } + /// Generate the wide load/store. + void execute(VPTransformState &State) override { + llvm_unreachable("VPWidenMemoryRecipe should not be instantiated."); + } - /// Return the address accessed by this recipe. - VPValue *getStoredValue() const { - assert(isStore() && "Stored value only available for store instructions"); - return getOperand(1); // Stored value is the 2nd, mandatory operand. + Instruction &getIngredient() const { return Ingredient; } +}; + +/// A recipe for widening load operations, using the address to load from and an +/// optional mask. +struct VPWidenLoadRecipe final : public VPWidenMemoryRecipe, public VPValue { + VPWidenLoadRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask, + bool Consecutive, bool Reverse, DebugLoc DL) + : VPWidenMemoryRecipe(VPDef::VPWidenLoadSC, Load, {Addr}, Consecutive, + Reverse, DL), + VPValue(this, &Load) { + setMask(Mask); } - // Return whether the loaded-from / stored-to addresses are consecutive. - bool isConsecutive() const { return Consecutive; } + VPWidenLoadRecipe *clone() override { + return new VPWidenLoadRecipe(cast(Ingredient), getAddr(), + getMask(), Consecutive, Reverse, + getDebugLoc()); + } - // Return whether the consecutive loaded/stored addresses are in reverse - // order. - bool isReverse() const { return Reverse; } + VP_CLASSOF_IMPL(VPDef::VPWidenLoadSC); - /// Generate the wide load/store. + /// Generate a wide load or gather. void execute(VPTransformState &State) override; #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) @@ -2380,16 +2386,51 @@ public: assert(is_contained(operands(), Op) && "Op must be an operand of the recipe"); - // Widened, consecutive memory operations only demand the first lane of - // their address, unless the same operand is also stored. That latter can - // happen with opaque pointers. - return Op == getAddr() && isConsecutive() && - (!isStore() || Op != getStoredValue()); + // Widened, consecutive loads operations only demand the first lane of + // their address. + return Op == getAddr() && isConsecutive(); } - - Instruction &getIngredient() const { return Ingredient; } }; +/// A recipe for widening store operations, using the stored value, the address +/// to store to and an optional mask. +struct VPWidenStoreRecipe final : public VPWidenMemoryRecipe { + VPWidenStoreRecipe(StoreInst &Store, VPValue *Addr, VPValue *StoredVal, + VPValue *Mask, bool Consecutive, bool Reverse, DebugLoc DL) + : VPWidenMemoryRecipe(VPDef::VPWidenStoreSC, Store, {Addr, StoredVal}, + Consecutive, Reverse, DL) { + setMask(Mask); + } + + VPWidenStoreRecipe *clone() override { + return new VPWidenStoreRecipe(cast(Ingredient), getAddr(), + getStoredValue(), getMask(), Consecutive, + Reverse, getDebugLoc()); + } + + VP_CLASSOF_IMPL(VPDef::VPWidenStoreSC); + + /// Return the value stored by this recipe. + VPValue *getStoredValue() const { return getOperand(1); } + + /// Generate a wide store or scatter. + void execute(VPTransformState &State) override; + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + /// Print the recipe. + void print(raw_ostream &O, const Twine &Indent, + VPSlotTracker &SlotTracker) const override; +#endif + + /// Returns true if the recipe only uses the first lane of operand \p Op. + bool onlyFirstLaneUsed(const VPValue *Op) const override { + assert(is_contained(operands(), Op) && + "Op must be an operand of the recipe"); + // Widened, consecutive stores only demand the first lane of their address, + // unless the same operand is also stored. + return Op == getAddr() && isConsecutive() && Op != getStoredValue(); + } +}; /// Recipe to expand a SCEV expression. class VPExpandSCEVRecipe : public VPSingleDefRecipe { const SCEV *Expr; diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp index c8ae2ee5a30f..130fb04f586e 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp @@ -108,9 +108,9 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPWidenCallRecipe *R) { return CI.getType(); } -Type *VPTypeAnalysis::inferScalarTypeForRecipe( - const VPWidenMemoryInstructionRecipe *R) { - assert(!R->isStore() && "Store recipes should not define any values"); +Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPWidenMemoryRecipe *R) { + assert(isa(R) && + "Store recipes should not define any values"); return cast(&R->getIngredient())->getType(); } @@ -231,8 +231,7 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) { return inferScalarType(R->getOperand(0)); }) .Case( + VPWidenCallRecipe, VPWidenMemoryRecipe, VPWidenSelectRecipe>( [this](const auto *R) { return inferScalarTypeForRecipe(R); }) .Case([V](const VPInterleaveRecipe *R) { // TODO: Use info from interleave group. diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h index 4e69de7fd681..7d310b1b31b6 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h +++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h @@ -20,7 +20,7 @@ class VPInstruction; class VPWidenRecipe; class VPWidenCallRecipe; class VPWidenIntOrFpInductionRecipe; -class VPWidenMemoryInstructionRecipe; +class VPWidenMemoryRecipe; struct VPWidenSelectRecipe; class VPReplicateRecipe; class Type; @@ -46,7 +46,7 @@ class VPTypeAnalysis { Type *inferScalarTypeForRecipe(const VPWidenCallRecipe *R); Type *inferScalarTypeForRecipe(const VPWidenRecipe *R); Type *inferScalarTypeForRecipe(const VPWidenIntOrFpInductionRecipe *R); - Type *inferScalarTypeForRecipe(const VPWidenMemoryInstructionRecipe *R); + Type *inferScalarTypeForRecipe(const VPWidenMemoryRecipe *R); Type *inferScalarTypeForRecipe(const VPWidenSelectRecipe *R); Type *inferScalarTypeForRecipe(const VPReplicateRecipe *R); diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 9f242a1bee8f..78932643c81f 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -47,9 +47,8 @@ bool VPRecipeBase::mayWriteToMemory() const { switch (getVPDefID()) { case VPInterleaveSC: return cast(this)->getNumStoreOperands() > 0; - case VPWidenMemoryInstructionSC: { - return cast(this)->isStore(); - } + case VPWidenStoreSC: + return true; case VPReplicateSC: case VPWidenCallSC: return cast(getVPSingleValue()->getUnderlyingValue()) @@ -64,6 +63,7 @@ bool VPRecipeBase::mayWriteToMemory() const { case VPWidenCastSC: case VPWidenGEPSC: case VPWidenIntOrFpInductionSC: + case VPWidenLoadSC: case VPWidenPHISC: case VPWidenSC: case VPWidenSelectSC: { @@ -81,16 +81,16 @@ bool VPRecipeBase::mayWriteToMemory() const { bool VPRecipeBase::mayReadFromMemory() const { switch (getVPDefID()) { - case VPWidenMemoryInstructionSC: { - return !cast(this)->isStore(); - } + case VPWidenLoadSC: + return true; case VPReplicateSC: case VPWidenCallSC: return cast(getVPSingleValue()->getUnderlyingValue()) ->mayReadFromMemory(); case VPBranchOnMaskSC: - case VPScalarIVStepsSC: case VPPredInstPHISC: + case VPScalarIVStepsSC: + case VPWidenStoreSC: return false; case VPBlendSC: case VPReductionSC: @@ -155,12 +155,13 @@ bool VPRecipeBase::mayHaveSideEffects() const { } case VPInterleaveSC: return mayWriteToMemory(); - case VPWidenMemoryInstructionSC: - assert(cast(this) - ->getIngredient() - .mayHaveSideEffects() == mayWriteToMemory() && - "mayHaveSideffects result for ingredient differs from this " - "implementation"); + case VPWidenLoadSC: + case VPWidenStoreSC: + assert( + cast(this)->getIngredient().mayHaveSideEffects() == + mayWriteToMemory() && + "mayHaveSideffects result for ingredient differs from this " + "implementation"); return mayWriteToMemory(); case VPReplicateSC: { auto *R = cast(this); @@ -1769,16 +1770,17 @@ void VPPredInstPHIRecipe::print(raw_ostream &O, const Twine &Indent, printOperands(O, SlotTracker); } -void VPWidenMemoryInstructionRecipe::print(raw_ostream &O, const Twine &Indent, - VPSlotTracker &SlotTracker) const { +void VPWidenLoadRecipe::print(raw_ostream &O, const Twine &Indent, + VPSlotTracker &SlotTracker) const { O << Indent << "WIDEN "; + printAsOperand(O, SlotTracker); + O << " = load "; + printOperands(O, SlotTracker); +} - if (!isStore()) { - getVPSingleValue()->printAsOperand(O, SlotTracker); - O << " = "; - } - O << Instruction::getOpcodeName(Ingredient.getOpcode()) << " "; - +void VPWidenStoreRecipe::print(raw_ostream &O, const Twine &Indent, + VPSlotTracker &SlotTracker) const { + O << Indent << "WIDEN store "; printOperands(O, SlotTracker); } #endif diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 1256e4d8fda5..382bf5ac1140 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -60,14 +60,14 @@ void VPlanTransforms::VPInstructionsToVPRecipes( assert(isa(&Ingredient) && "only VPInstructions expected here"); assert(!isa(Inst) && "phis should be handled above"); - // Create VPWidenMemoryInstructionRecipe for loads and stores. + // Create VPWidenMemoryRecipe for loads and stores. if (LoadInst *Load = dyn_cast(Inst)) { - NewRecipe = new VPWidenMemoryInstructionRecipe( + NewRecipe = new VPWidenLoadRecipe( *Load, Ingredient.getOperand(0), nullptr /*Mask*/, false /*Consecutive*/, false /*Reverse*/, Ingredient.getDebugLoc()); } else if (StoreInst *Store = dyn_cast(Inst)) { - NewRecipe = new VPWidenMemoryInstructionRecipe( + NewRecipe = new VPWidenStoreRecipe( *Store, Ingredient.getOperand(1), Ingredient.getOperand(0), nullptr /*Mask*/, false /*Consecutive*/, false /*Reverse*/, Ingredient.getDebugLoc()); @@ -977,10 +977,9 @@ void VPlanTransforms::truncateToMinimalBitwidths( vp_depth_first_deep(Plan.getVectorLoopRegion()))) { for (VPRecipeBase &R : make_early_inc_range(*VPBB)) { if (!isa(&R)) + VPWidenSelectRecipe, VPWidenMemoryRecipe>(&R)) continue; - if (isa(&R) && - cast(&R)->isStore()) + if (isa(&R)) continue; VPValue *ResultVPV = R.getVPSingleValue(); @@ -1048,10 +1047,9 @@ void VPlanTransforms::truncateToMinimalBitwidths( assert(cast(&R)->getOpcode() == Instruction::ICmp && "Only ICmps should not need extending the result."); - if (isa(&R)) { - assert(!cast(&R)->isStore() && "stores cannot be narrowed"); + assert(!isa(&R) && "stores cannot be narrowed"); + if (isa(&R)) continue; - } // Shrink operands by introducing truncates as needed. unsigned StartIdx = isa(&R) ? 1 : 0; @@ -1315,7 +1313,7 @@ void VPlanTransforms::addExplicitVectorLength(VPlan &Plan) { ConstantInt::getTrue(CanonicalIVPHI->getScalarType()->getContext()); VPValue *VPTrueMask = Plan.getOrAddLiveIn(TrueMask); replaceHeaderPredicateWith(Plan, *VPTrueMask, [](VPUser &U, unsigned) { - return isa(U); + return isa(U); }); // Now create the ExplicitVectorLengthPhi recipe in the main loop. auto *EVLPhi = new VPEVLBasedIVPHIRecipe(StartV, DebugLoc()); @@ -1371,8 +1369,7 @@ void VPlanTransforms::dropPoisonGeneratingRecipes( // instruction. Widen memory instructions involved in address computation // will lead to gather/scatter instructions, which don't need to be // handled. - if (isa(CurRec) || - isa(CurRec) || + if (isa(CurRec) || isa(CurRec) || isa(CurRec) || isa(CurRec)) continue; @@ -1420,7 +1417,7 @@ void VPlanTransforms::dropPoisonGeneratingRecipes( auto Iter = vp_depth_first_deep(Plan.getEntry()); for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly(Iter)) { for (VPRecipeBase &Recipe : *VPBB) { - if (auto *WidenRec = dyn_cast(&Recipe)) { + if (auto *WidenRec = dyn_cast(&Recipe)) { Instruction &UnderlyingInstr = WidenRec->getIngredient(); VPRecipeBase *AddrDef = WidenRec->getAddr()->getDefiningRecipe(); if (AddrDef && WidenRec->isConsecutive() && diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h index 3f8d4f4fe7d6..0bbc7ffb4a2f 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -36,7 +36,6 @@ class VPDef; class VPSlotTracker; class VPUser; class VPRecipeBase; -class VPWidenMemoryInstructionRecipe; // This is the base class of the VPlan Def/Use graph, used for modeling the data // flow into, within and out of the VPlan. VPValues can stand for live-ins @@ -51,7 +50,6 @@ class VPValue { friend class VPInterleavedAccessInfo; friend class VPSlotTracker; friend class VPRecipeBase; - friend class VPWidenMemoryInstructionRecipe; const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast). @@ -358,7 +356,8 @@ public: VPWidenCanonicalIVSC, VPWidenCastSC, VPWidenGEPSC, - VPWidenMemoryInstructionSC, + VPWidenLoadSC, + VPWidenStoreSC, VPWidenSC, VPWidenSelectSC, VPBlendSC, diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp index 12d37fa711db..5587302207ac 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp @@ -128,7 +128,7 @@ static bool verifyVPBasicBlock(const VPBasicBlock *VPBB, } return true; } - if (isa(R)) + if (isa(R)) VPWidenMemRecipe = R; return true; }; diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse-mask4.ll b/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse-mask4.ll index d5ace655fdcc..c22613509be4 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse-mask4.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse-mask4.ll @@ -46,8 +46,8 @@ define void @vector_reverse_mask_v4i1(ptr noalias %a, ptr noalias %cond, i64 %N) ; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i64 -24 ; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP7]], i64 -56 ; CHECK-NEXT: [[REVERSE3:%.*]] = shufflevector <4 x i1> [[TMP5]], <4 x i1> poison, <4 x i32> -; CHECK-NEXT: [[REVERSE4:%.*]] = shufflevector <4 x i1> [[TMP6]], <4 x i1> poison, <4 x i32> ; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr [[TMP8]], i32 8, <4 x i1> [[REVERSE3]], <4 x double> poison) +; CHECK-NEXT: [[REVERSE4:%.*]] = shufflevector <4 x i1> [[TMP6]], <4 x i1> poison, <4 x i32> ; CHECK-NEXT: [[WIDE_MASKED_LOAD6:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr [[TMP9]], i32 8, <4 x i1> [[REVERSE4]], <4 x double> poison) ; CHECK-NEXT: [[TMP10:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD]], ; CHECK-NEXT: [[TMP11:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD6]], diff --git a/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll b/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll index eea2894f8279..aea72b7de5f4 100644 --- a/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll @@ -1400,15 +1400,15 @@ define void @foo6(ptr nocapture readonly %in, ptr nocapture %out, i32 %size, ptr ; AVX2-NEXT: [[TMP30:%.*]] = getelementptr double, ptr [[TMP20]], i32 -12 ; AVX2-NEXT: [[TMP31:%.*]] = getelementptr double, ptr [[TMP30]], i32 -3 ; AVX2-NEXT: [[REVERSE12:%.*]] = shufflevector <4 x i1> [[TMP16]], <4 x i1> poison, <4 x i32> -; AVX2-NEXT: [[REVERSE14:%.*]] = shufflevector <4 x i1> [[TMP17]], <4 x i1> poison, <4 x i32> -; AVX2-NEXT: [[REVERSE17:%.*]] = shufflevector <4 x i1> [[TMP18]], <4 x i1> poison, <4 x i32> -; AVX2-NEXT: [[REVERSE20:%.*]] = shufflevector <4 x i1> [[TMP19]], <4 x i1> poison, <4 x i32> ; AVX2-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr [[TMP25]], i32 8, <4 x i1> [[REVERSE12]], <4 x double> poison), !alias.scope !21 ; AVX2-NEXT: [[REVERSE13:%.*]] = shufflevector <4 x double> [[WIDE_MASKED_LOAD]], <4 x double> poison, <4 x i32> +; AVX2-NEXT: [[REVERSE14:%.*]] = shufflevector <4 x i1> [[TMP17]], <4 x i1> poison, <4 x i32> ; AVX2-NEXT: [[WIDE_MASKED_LOAD15:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr [[TMP27]], i32 8, <4 x i1> [[REVERSE14]], <4 x double> poison), !alias.scope !21 ; AVX2-NEXT: [[REVERSE16:%.*]] = shufflevector <4 x double> [[WIDE_MASKED_LOAD15]], <4 x double> poison, <4 x i32> +; AVX2-NEXT: [[REVERSE17:%.*]] = shufflevector <4 x i1> [[TMP18]], <4 x i1> poison, <4 x i32> ; AVX2-NEXT: [[WIDE_MASKED_LOAD18:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr [[TMP29]], i32 8, <4 x i1> [[REVERSE17]], <4 x double> poison), !alias.scope !21 ; AVX2-NEXT: [[REVERSE19:%.*]] = shufflevector <4 x double> [[WIDE_MASKED_LOAD18]], <4 x double> poison, <4 x i32> +; AVX2-NEXT: [[REVERSE20:%.*]] = shufflevector <4 x i1> [[TMP19]], <4 x i1> poison, <4 x i32> ; AVX2-NEXT: [[WIDE_MASKED_LOAD21:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr [[TMP31]], i32 8, <4 x i1> [[REVERSE20]], <4 x double> poison), !alias.scope !21 ; AVX2-NEXT: [[REVERSE22:%.*]] = shufflevector <4 x double> [[WIDE_MASKED_LOAD21]], <4 x double> poison, <4 x i32> ; AVX2-NEXT: [[TMP32:%.*]] = fadd <4 x double> [[REVERSE13]], @@ -1524,15 +1524,15 @@ define void @foo6(ptr nocapture readonly %in, ptr nocapture %out, i32 %size, ptr ; AVX512-NEXT: [[TMP30:%.*]] = getelementptr double, ptr [[TMP20]], i32 -24 ; AVX512-NEXT: [[TMP31:%.*]] = getelementptr double, ptr [[TMP30]], i32 -7 ; AVX512-NEXT: [[REVERSE12:%.*]] = shufflevector <8 x i1> [[TMP16]], <8 x i1> poison, <8 x i32> -; AVX512-NEXT: [[REVERSE14:%.*]] = shufflevector <8 x i1> [[TMP17]], <8 x i1> poison, <8 x i32> -; AVX512-NEXT: [[REVERSE17:%.*]] = shufflevector <8 x i1> [[TMP18]], <8 x i1> poison, <8 x i32> -; AVX512-NEXT: [[REVERSE20:%.*]] = shufflevector <8 x i1> [[TMP19]], <8 x i1> poison, <8 x i32> ; AVX512-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0(ptr [[TMP25]], i32 8, <8 x i1> [[REVERSE12]], <8 x double> poison), !alias.scope !34 ; AVX512-NEXT: [[REVERSE13:%.*]] = shufflevector <8 x double> [[WIDE_MASKED_LOAD]], <8 x double> poison, <8 x i32> +; AVX512-NEXT: [[REVERSE14:%.*]] = shufflevector <8 x i1> [[TMP17]], <8 x i1> poison, <8 x i32> ; AVX512-NEXT: [[WIDE_MASKED_LOAD15:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0(ptr [[TMP27]], i32 8, <8 x i1> [[REVERSE14]], <8 x double> poison), !alias.scope !34 ; AVX512-NEXT: [[REVERSE16:%.*]] = shufflevector <8 x double> [[WIDE_MASKED_LOAD15]], <8 x double> poison, <8 x i32> +; AVX512-NEXT: [[REVERSE17:%.*]] = shufflevector <8 x i1> [[TMP18]], <8 x i1> poison, <8 x i32> ; AVX512-NEXT: [[WIDE_MASKED_LOAD18:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0(ptr [[TMP29]], i32 8, <8 x i1> [[REVERSE17]], <8 x double> poison), !alias.scope !34 ; AVX512-NEXT: [[REVERSE19:%.*]] = shufflevector <8 x double> [[WIDE_MASKED_LOAD18]], <8 x double> poison, <8 x i32> +; AVX512-NEXT: [[REVERSE20:%.*]] = shufflevector <8 x i1> [[TMP19]], <8 x i1> poison, <8 x i32> ; AVX512-NEXT: [[WIDE_MASKED_LOAD21:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0(ptr [[TMP31]], i32 8, <8 x i1> [[REVERSE20]], <8 x double> poison), !alias.scope !34 ; AVX512-NEXT: [[REVERSE22:%.*]] = shufflevector <8 x double> [[WIDE_MASKED_LOAD21]], <8 x double> poison, <8 x i32> ; AVX512-NEXT: [[TMP32:%.*]] = fadd <8 x double> [[REVERSE13]], diff --git a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp index 777675b623f3..2b25c62ac2f6 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp @@ -192,9 +192,9 @@ TEST_F(VPlanHCFGTest, testVPInstructionToVPRecipesInner) { auto Iter = VecBB->begin(); EXPECT_NE(nullptr, dyn_cast(&*Iter++)); EXPECT_NE(nullptr, dyn_cast(&*Iter++)); - EXPECT_NE(nullptr, dyn_cast(&*Iter++)); + EXPECT_NE(nullptr, dyn_cast(&*Iter++)); EXPECT_NE(nullptr, dyn_cast(&*Iter++)); - EXPECT_NE(nullptr, dyn_cast(&*Iter++)); + EXPECT_NE(nullptr, dyn_cast(&*Iter++)); EXPECT_NE(nullptr, dyn_cast(&*Iter++)); EXPECT_NE(nullptr, dyn_cast(&*Iter++)); EXPECT_NE(nullptr, dyn_cast(&*Iter++)); diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp index cb8737a9e64d..64e9c06db3fe 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp @@ -1029,7 +1029,7 @@ TEST(VPRecipeTest, CastVPBranchOnMaskRecipeToVPUser) { EXPECT_EQ(&Recipe, BaseR); } -TEST(VPRecipeTest, CastVPWidenMemoryInstructionRecipeToVPUserAndVPDef) { +TEST(VPRecipeTest, CastVPWidenMemoryRecipeToVPUserAndVPDef) { LLVMContext C; IntegerType *Int32 = IntegerType::get(C, 32); @@ -1038,7 +1038,7 @@ TEST(VPRecipeTest, CastVPWidenMemoryInstructionRecipeToVPUserAndVPDef) { new LoadInst(Int32, UndefValue::get(Int32Ptr), "", false, Align(1)); VPValue Addr; VPValue Mask; - VPWidenMemoryInstructionRecipe Recipe(*Load, &Addr, &Mask, true, false, {}); + VPWidenLoadRecipe Recipe(*Load, &Addr, &Mask, true, false, {}); EXPECT_TRUE(isa(&Recipe)); VPRecipeBase *BaseR = &Recipe; EXPECT_TRUE(isa(BaseR)); @@ -1133,7 +1133,7 @@ TEST(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) { new LoadInst(Int32, UndefValue::get(Int32Ptr), "", false, Align(1)); VPValue Addr; VPValue Mask; - VPWidenMemoryInstructionRecipe Recipe(*Load, &Addr, &Mask, true, false, {}); + VPWidenLoadRecipe Recipe(*Load, &Addr, &Mask, true, false, {}); EXPECT_FALSE(Recipe.mayHaveSideEffects()); EXPECT_TRUE(Recipe.mayReadFromMemory()); EXPECT_FALSE(Recipe.mayWriteToMemory()); @@ -1147,8 +1147,7 @@ TEST(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) { VPValue Addr; VPValue Mask; VPValue StoredV; - VPWidenMemoryInstructionRecipe Recipe(*Store, &Addr, &StoredV, &Mask, false, - false, {}); + VPWidenStoreRecipe Recipe(*Store, &Addr, &StoredV, &Mask, false, false, {}); EXPECT_TRUE(Recipe.mayHaveSideEffects()); EXPECT_FALSE(Recipe.mayReadFromMemory()); EXPECT_TRUE(Recipe.mayWriteToMemory()); -- GitLab From f4737a2edd900df661750116821806bb45e4086a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolai=20H=C3=A4hnle?= Date: Wed, 17 Apr 2024 12:24:32 +0200 Subject: [PATCH 029/862] update_test_checks: keep names stable with generated functions (#87988) Collect the original check lines in a manner that is independent of where the check lines appear in the file. This is so that we keep FileCheck variable names stable even when --include-generated-funcs is used. Reported-by: Ruiling Song --- .../Inputs/stable_ir_values_funcs.ll.expected | 10 ++-- llvm/utils/UpdateTestChecks/common.py | 59 +++++++++++-------- 2 files changed, 40 insertions(+), 29 deletions(-) diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values_funcs.ll.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values_funcs.ll.expected index 1559319ac013..86f929ffe36a 100644 --- a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values_funcs.ll.expected +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values_funcs.ll.expected @@ -16,9 +16,9 @@ define i32 @func({i32, i32} %x, i32 %y) { ; CHECK-LABEL: define i32 @func( ; CHECK-SAME: { i32, i32 } [[X:%.*]], i32 [[Y:%.*]]) { -; CHECK-NEXT: [[X_I34:%.*]] = extractvalue { i32, i32 } [[X]], 0 -; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[Y]], 1 -; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[X_I34]], [[TMP1]] -; CHECK-NEXT: [[TMP3:%.*]] = mul i32 [[TMP2]], 3 -; CHECK-NEXT: ret i32 [[TMP3]] +; CHECK-NEXT: [[X_I33:%.*]] = extractvalue { i32, i32 } [[X]], 0 +; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[Y]], 1 +; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[X_I33]], [[TMP3]] +; CHECK-NEXT: [[TMP2:%.*]] = mul i32 [[TMP1]], 3 +; CHECK-NEXT: ret i32 [[TMP2]] ; diff --git a/llvm/utils/UpdateTestChecks/common.py b/llvm/utils/UpdateTestChecks/common.py index eed36a0cdd73..15d3d5e527d6 100644 --- a/llvm/utils/UpdateTestChecks/common.py +++ b/llvm/utils/UpdateTestChecks/common.py @@ -430,36 +430,47 @@ def collect_original_check_lines(ti: TestInfo, prefix_set: set): result[func_name][prefix] is filled with a list of right-hand-sides of check lines. """ - result = {} + result = collections.defaultdict(lambda: {}) + current_prefix = None current_function = None for input_line_info in ti.ro_iterlines(): input_line = input_line_info.line - if current_function is not None: - if input_line == "": - continue - if input_line.lstrip().startswith(";"): - m = CHECK_RE.match(input_line) - if ( - m is not None - and m.group(1) in prefix_set - and m.group(2) not in ["LABEL", "SAME"] - ): - if m.group(1) not in current_function: - current_function[m.group(1)] = [] - current_function[m.group(1)].append(input_line[m.end() :].strip()) - continue - current_function = None + if input_line.lstrip().startswith(";"): + m = CHECK_RE.match(input_line) + if m is not None: + prefix = m.group(1) + check_kind = m.group(2) + line = input_line[m.end() :].strip() + + if prefix != current_prefix: + current_function = None + current_prefix = None + + if check_kind not in ["LABEL", "SAME"]: + if current_function is not None: + current_function.append(line) + continue - m = IR_FUNCTION_RE.match(input_line) - if m is not None: - func_name = m.group(1) - if ti.args.function is not None and func_name != ti.args.function: - # When filtering on a specific function, skip all others. - continue + if check_kind == "SAME": + continue + + if check_kind == "LABEL": + m = IR_FUNCTION_RE.match(line) + if m is not None: + func_name = m.group(1) + if ( + ti.args.function is not None + and func_name != ti.args.function + ): + # When filtering on a specific function, skip all others. + continue + + current_prefix = prefix + current_function = result[func_name][prefix] = [] + continue - assert func_name not in result - current_function = result[func_name] = {} + current_function = None return result -- GitLab From 3eb0ba34b0a2a29c2f34ead2b84fdf9b62cb29c1 Mon Sep 17 00:00:00 2001 From: Sergio Afonso Date: Wed, 17 Apr 2024 11:28:30 +0100 Subject: [PATCH 030/862] [MLIR][Flang][OpenMP] Make omp.simdloop into a loop wrapper (#87365) This patch updates the definition of `omp.simdloop` to enforce the restrictions of a wrapper operation. It has been renamed to `omp.simd`, to better reflect the naming used in the spec. All uses of "simdloop" in function names have been updated accordingly. Some changes to Flang lowering and OpenMP to LLVM IR translation are introduced to prevent the introduction of compilation/test failures. The eventual long term solution might be different. --- flang/lib/Lower/OpenMP/OpenMP.cpp | 167 +++++++++----- .../Fir/convert-to-llvm-openmp-and-fir.fir | 101 +++++---- flang/test/Lower/OpenMP/FIR/if-clause.f90 | 23 +- flang/test/Lower/OpenMP/FIR/loop-combined.f90 | 2 +- .../OpenMP/FIR/parallel-private-clause.f90 | 3 +- flang/test/Lower/OpenMP/FIR/simd.f90 | 109 +++++---- flang/test/Lower/OpenMP/if-clause.f90 | 23 +- flang/test/Lower/OpenMP/loop-combined.f90 | 2 +- .../Lower/OpenMP/parallel-private-clause.f90 | 3 +- flang/test/Lower/OpenMP/simd.f90 | 123 +++++----- .../Frontend/OpenMPIRBuilderTest.cpp | 2 +- .../Dialect/OpenMP/OpenMPClauseOperands.h | 9 +- mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td | 58 ++--- .../Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp | 17 +- mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp | 45 ++-- .../OpenMP/OpenMPToLLVMIRTranslation.cpp | 41 ++-- .../OpenMPToLLVM/convert-to-llvmir.mlir | 31 +-- mlir/test/Dialect/OpenMP/invalid.mlir | 163 +++++++------ mlir/test/Dialect/OpenMP/ops.mlir | 214 +++++++++--------- mlir/test/Target/LLVMIR/openmp-llvm.mlir | 157 +++++++------ 20 files changed, 701 insertions(+), 592 deletions(-) diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp index 9b9975223666..c31d63625dbb 100644 --- a/flang/lib/Lower/OpenMP/OpenMP.cpp +++ b/flang/lib/Lower/OpenMP/OpenMP.cpp @@ -502,8 +502,10 @@ struct OpWithBodyGenInfo { OpWithBodyGenInfo(Fortran::lower::AbstractConverter &converter, Fortran::semantics::SemanticsContext &semaCtx, - mlir::Location loc, Fortran::lower::pft::Evaluation &eval) - : converter(converter), semaCtx(semaCtx), loc(loc), eval(eval) {} + mlir::Location loc, Fortran::lower::pft::Evaluation &eval, + llvm::omp::Directive dir) + : converter(converter), semaCtx(semaCtx), loc(loc), eval(eval), dir(dir) { + } OpWithBodyGenInfo &setGenNested(bool value) { genNested = value; @@ -546,6 +548,8 @@ struct OpWithBodyGenInfo { mlir::Location loc; /// [in] current PFT node/evaluation. Fortran::lower::pft::Evaluation &eval; + /// [in] leaf directive for which to generate the op body. + llvm::omp::Directive dir; /// [in] whether to generate FIR for nested evaluations bool genNested = true; /// [in] is this an outer operation - prevents privatization. @@ -568,8 +572,7 @@ struct OpWithBodyGenInfo { /// /// \param [in] op - the operation the body belongs to. /// \param [in] info - options controlling code-gen for the construction. -template -static void createBodyOfOp(Op &op, OpWithBodyGenInfo &info) { +static void createBodyOfOp(mlir::Operation &op, OpWithBodyGenInfo &info) { fir::FirOpBuilder &firOpBuilder = info.converter.getFirOpBuilder(); auto insertMarker = [](fir::FirOpBuilder &builder) { @@ -585,10 +588,10 @@ static void createBodyOfOp(Op &op, OpWithBodyGenInfo &info) { auto regionArgs = [&]() -> llvm::SmallVector { if (info.genRegionEntryCB != nullptr) { - return info.genRegionEntryCB(op); + return info.genRegionEntryCB(&op); } - firOpBuilder.createBlock(&op.getRegion()); + firOpBuilder.createBlock(&op.getRegion(0)); return {}; }(); // Mark the earliest insertion point. @@ -603,8 +606,8 @@ static void createBodyOfOp(Op &op, OpWithBodyGenInfo &info) { // Start with privatization, so that the lowering of the nested // code will use the right symbols. - constexpr bool isLoop = std::is_same_v || - std::is_same_v; + bool isLoop = llvm::omp::getDirectiveAssociation(info.dir) == + llvm::omp::Association::Loop; bool privatize = info.clauses && !info.outerCombined; firOpBuilder.setInsertionPoint(marker); @@ -616,7 +619,7 @@ static void createBodyOfOp(Op &op, OpWithBodyGenInfo &info) { } } - if constexpr (std::is_same_v) { + if (info.dir == llvm::omp::Directive::OMPD_parallel) { threadPrivatizeVars(info.converter, info.eval); if (info.clauses) { firOpBuilder.setInsertionPoint(marker); @@ -630,9 +633,9 @@ static void createBodyOfOp(Op &op, OpWithBodyGenInfo &info) { // a lot of complications for our approach if the terminator generation // is delayed past this point. Insert a temporary terminator here, then // delete it. - firOpBuilder.setInsertionPointToEnd(&op.getRegion().back()); - auto *temp = Fortran::lower::genOpenMPTerminator( - firOpBuilder, op.getOperation(), info.loc); + firOpBuilder.setInsertionPointToEnd(&op.getRegion(0).back()); + auto *temp = + Fortran::lower::genOpenMPTerminator(firOpBuilder, &op, info.loc); firOpBuilder.setInsertionPointAfter(marker); genNestedEvaluations(info.converter, info.eval); temp->erase(); @@ -674,23 +677,36 @@ static void createBodyOfOp(Op &op, OpWithBodyGenInfo &info) { return exit; }; - if (auto *exitBlock = getUniqueExit(op.getRegion())) { + if (auto *exitBlock = getUniqueExit(op.getRegion(0))) { firOpBuilder.setInsertionPointToEnd(exitBlock); - auto *term = Fortran::lower::genOpenMPTerminator( - firOpBuilder, op.getOperation(), info.loc); + auto *term = + Fortran::lower::genOpenMPTerminator(firOpBuilder, &op, info.loc); // Only insert lastprivate code when there actually is an exit block. // Such a block may not exist if the nested code produced an infinite // loop (this may not make sense in production code, but a user could // write that and we should handle it). firOpBuilder.setInsertionPoint(term); if (privatize) { + // DataSharingProcessor::processStep2() may create operations before/after + // the one passed as argument. We need to treat loop wrappers and their + // nested loop as a unit, so we need to pass the top level wrapper (if + // present). Otherwise, these operations will be inserted within a + // wrapper region. + mlir::Operation *privatizationTopLevelOp = &op; + if (auto loopNest = llvm::dyn_cast(op)) { + llvm::SmallVector wrappers; + loopNest.gatherWrappers(wrappers); + if (!wrappers.empty()) + privatizationTopLevelOp = &*wrappers.back(); + } + if (!info.dsp) { assert(tempDsp.has_value()); - tempDsp->processStep2(op, isLoop); + tempDsp->processStep2(privatizationTopLevelOp, isLoop); } else { if (isLoop && regionArgs.size() > 0) info.dsp->setLoopIV(info.converter.getSymbolAddress(*regionArgs[0])); - info.dsp->processStep2(op, isLoop); + info.dsp->processStep2(privatizationTopLevelOp, isLoop); } } } @@ -921,7 +937,7 @@ template static OpTy genOpWithBody(OpWithBodyGenInfo &info, Args &&...args) { auto op = info.converter.getFirOpBuilder().create( info.loc, std::forward(args)...); - createBodyOfOp(op, info); + createBodyOfOp(*op, info); return op; } @@ -954,6 +970,18 @@ static void genFlushClauses( TODO(converter.getCurrentLocation(), "Handle OmpMemoryOrderClause"); } +static void genLoopNestClauses( + Fortran::lower::AbstractConverter &converter, + Fortran::semantics::SemanticsContext &semaCtx, + Fortran::lower::pft::Evaluation &eval, + const Fortran::parser::OmpClauseList &clauses, mlir::Location loc, + mlir::omp::LoopNestClauseOps &clauseOps, + llvm::SmallVectorImpl &iv) { + ClauseProcessor cp(converter, semaCtx, clauses); + cp.processCollapse(loc, eval, clauseOps, iv); + clauseOps.loopInclusiveAttr = converter.getFirOpBuilder().getUnitAttr(); +} + static void genOrderedRegionClauses(Fortran::lower::AbstractConverter &converter, Fortran::semantics::SemanticsContext &semaCtx, @@ -1002,21 +1030,16 @@ static void genSectionsClauses(Fortran::lower::AbstractConverter &converter, } } -static void genSimdLoopClauses( - Fortran::lower::AbstractConverter &converter, - Fortran::semantics::SemanticsContext &semaCtx, - Fortran::lower::StatementContext &stmtCtx, - Fortran::lower::pft::Evaluation &eval, - const Fortran::parser::OmpClauseList &clauses, mlir::Location loc, - mlir::omp::SimdLoopClauseOps &clauseOps, - llvm::SmallVectorImpl &iv) { +static void genSimdClauses(Fortran::lower::AbstractConverter &converter, + Fortran::semantics::SemanticsContext &semaCtx, + const Fortran::parser::OmpClauseList &clauses, + mlir::Location loc, + mlir::omp::SimdClauseOps &clauseOps) { ClauseProcessor cp(converter, semaCtx, clauses); - cp.processCollapse(loc, eval, clauseOps, iv); cp.processIf(llvm::omp::Directive::OMPD_simd, clauseOps); cp.processReduction(loc, clauseOps); cp.processSafelen(clauseOps); cp.processSimdlen(clauseOps); - clauseOps.loopInclusiveAttr = converter.getFirOpBuilder().getUnitAttr(); // TODO Support delayed privatization. cp.processTODO( - OpWithBodyGenInfo(converter, semaCtx, loc, eval).setGenNested(genNested), + OpWithBodyGenInfo(converter, semaCtx, loc, eval, + llvm::omp::Directive::OMPD_critical) + .setGenNested(genNested), nameAttr); } @@ -1295,7 +1320,9 @@ genMasterOp(Fortran::lower::AbstractConverter &converter, Fortran::lower::pft::Evaluation &eval, bool genNested, mlir::Location loc) { return genOpWithBody( - OpWithBodyGenInfo(converter, semaCtx, loc, eval).setGenNested(genNested)); + OpWithBodyGenInfo(converter, semaCtx, loc, eval, + llvm::omp::Directive::OMPD_master) + .setGenNested(genNested)); } static mlir::omp::OrderedOp @@ -1317,7 +1344,9 @@ genOrderedRegionOp(Fortran::lower::AbstractConverter &converter, genOrderedRegionClauses(converter, semaCtx, clauseList, loc, clauseOps); return genOpWithBody( - OpWithBodyGenInfo(converter, semaCtx, loc, eval).setGenNested(genNested), + OpWithBodyGenInfo(converter, semaCtx, loc, eval, + llvm::omp::Directive::OMPD_ordered) + .setGenNested(genNested), clauseOps); } @@ -1345,7 +1374,8 @@ genParallelOp(Fortran::lower::AbstractConverter &converter, }; OpWithBodyGenInfo genInfo = - OpWithBodyGenInfo(converter, semaCtx, loc, eval) + OpWithBodyGenInfo(converter, semaCtx, loc, eval, + llvm::omp::Directive::OMPD_parallel) .setGenNested(genNested) .setOuterCombined(outerCombined) .setClauses(&clauseList) @@ -1408,7 +1438,8 @@ genSectionOp(Fortran::lower::AbstractConverter &converter, // Currently only private/firstprivate clause is handled, and // all privatization is done within `omp.section` operations. return genOpWithBody( - OpWithBodyGenInfo(converter, semaCtx, loc, eval) + OpWithBodyGenInfo(converter, semaCtx, loc, eval, + llvm::omp::Directive::OMPD_section) .setGenNested(genNested) .setClauses(&clauseList)); } @@ -1419,23 +1450,39 @@ genSectionsOp(Fortran::lower::AbstractConverter &converter, Fortran::lower::pft::Evaluation &eval, mlir::Location loc, const mlir::omp::SectionsClauseOps &clauseOps) { return genOpWithBody( - OpWithBodyGenInfo(converter, semaCtx, loc, eval).setGenNested(false), + OpWithBodyGenInfo(converter, semaCtx, loc, eval, + llvm::omp::Directive::OMPD_sections) + .setGenNested(false), clauseOps); } -static mlir::omp::SimdLoopOp -genSimdLoopOp(Fortran::lower::AbstractConverter &converter, - Fortran::semantics::SemanticsContext &semaCtx, - Fortran::lower::pft::Evaluation &eval, mlir::Location loc, - const Fortran::parser::OmpClauseList &clauseList) { +static mlir::omp::SimdOp +genSimdOp(Fortran::lower::AbstractConverter &converter, + Fortran::semantics::SemanticsContext &semaCtx, + Fortran::lower::pft::Evaluation &eval, mlir::Location loc, + const Fortran::parser::OmpClauseList &clauseList) { + fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder(); DataSharingProcessor dsp(converter, semaCtx, clauseList, eval); dsp.processStep1(); Fortran::lower::StatementContext stmtCtx; - mlir::omp::SimdLoopClauseOps clauseOps; + mlir::omp::LoopNestClauseOps loopClauseOps; + mlir::omp::SimdClauseOps simdClauseOps; llvm::SmallVector iv; - genSimdLoopClauses(converter, semaCtx, stmtCtx, eval, clauseList, loc, - clauseOps, iv); + genLoopNestClauses(converter, semaCtx, eval, clauseList, loc, loopClauseOps, + iv); + genSimdClauses(converter, semaCtx, clauseList, loc, simdClauseOps); + + // Create omp.simd wrapper. + auto simdOp = firOpBuilder.create(loc, simdClauseOps); + + // TODO: Add reduction-related arguments to the wrapper's entry block. + firOpBuilder.createBlock(&simdOp.getRegion()); + firOpBuilder.setInsertionPoint( + Fortran::lower::genOpenMPTerminator(firOpBuilder, simdOp, loc)); + + // Create nested omp.loop_nest and fill body with loop contents. + auto loopOp = firOpBuilder.create(loc, loopClauseOps); auto *nestedEval = getCollapsedLoopEval(eval, Fortran::lower::getCollapseValue(clauseList)); @@ -1444,12 +1491,14 @@ genSimdLoopOp(Fortran::lower::AbstractConverter &converter, return genLoopVars(op, converter, loc, iv); }; - return genOpWithBody( - OpWithBodyGenInfo(converter, semaCtx, loc, *nestedEval) - .setClauses(&clauseList) - .setDataSharingProcessor(&dsp) - .setGenRegionEntryCb(ivCallback), - clauseOps); + createBodyOfOp(*loopOp, + OpWithBodyGenInfo(converter, semaCtx, loc, *nestedEval, + llvm::omp::Directive::OMPD_simd) + .setClauses(&clauseList) + .setDataSharingProcessor(&dsp) + .setGenRegionEntryCb(ivCallback)); + + return simdOp; } static mlir::omp::SingleOp @@ -1464,7 +1513,8 @@ genSingleOp(Fortran::lower::AbstractConverter &converter, clauseOps); return genOpWithBody( - OpWithBodyGenInfo(converter, semaCtx, loc, eval) + OpWithBodyGenInfo(converter, semaCtx, loc, eval, + llvm::omp::Directive::OMPD_single) .setGenNested(genNested) .setClauses(&beginClauseList), clauseOps); @@ -1645,7 +1695,8 @@ genTaskOp(Fortran::lower::AbstractConverter &converter, genTaskClauses(converter, semaCtx, stmtCtx, clauseList, loc, clauseOps); return genOpWithBody( - OpWithBodyGenInfo(converter, semaCtx, loc, eval) + OpWithBodyGenInfo(converter, semaCtx, loc, eval, + llvm::omp::Directive::OMPD_task) .setGenNested(genNested) .setClauses(&clauseList), clauseOps); @@ -1661,7 +1712,8 @@ genTaskgroupOp(Fortran::lower::AbstractConverter &converter, genTaskgroupClauses(converter, semaCtx, clauseList, loc, clauseOps); return genOpWithBody( - OpWithBodyGenInfo(converter, semaCtx, loc, eval) + OpWithBodyGenInfo(converter, semaCtx, loc, eval, + llvm::omp::Directive::OMPD_taskgroup) .setGenNested(genNested) .setClauses(&clauseList), clauseOps); @@ -1704,7 +1756,8 @@ genTeamsOp(Fortran::lower::AbstractConverter &converter, genTeamsClauses(converter, semaCtx, stmtCtx, clauseList, loc, clauseOps); return genOpWithBody( - OpWithBodyGenInfo(converter, semaCtx, loc, eval) + OpWithBodyGenInfo(converter, semaCtx, loc, eval, + llvm::omp::Directive::OMPD_teams) .setGenNested(genNested) .setOuterCombined(outerCombined) .setClauses(&clauseList), @@ -1738,7 +1791,8 @@ genWsloopOp(Fortran::lower::AbstractConverter &converter, }; return genOpWithBody( - OpWithBodyGenInfo(converter, semaCtx, loc, *nestedEval) + OpWithBodyGenInfo(converter, semaCtx, loc, *nestedEval, + llvm::omp::Directive::OMPD_do) .setClauses(&beginClauseList) .setDataSharingProcessor(&dsp) .setReductions(&reductionSyms, &reductionTypes) @@ -2253,7 +2307,7 @@ static void genOMP(Fortran::lower::AbstractConverter &converter, endClauseList, currentLocation); } else if (llvm::omp::allSimdSet.test(ompDirective)) { // 2.9.3.1 SIMD construct - genSimdLoopOp(converter, semaCtx, eval, currentLocation, beginClauseList); + genSimdOp(converter, semaCtx, eval, currentLocation, beginClauseList); } else { genWsloopOp(converter, semaCtx, eval, currentLocation, beginClauseList, endClauseList); @@ -2341,10 +2395,9 @@ mlir::Operation *Fortran::lower::genOpenMPTerminator(fir::FirOpBuilder &builder, mlir::Operation *op, mlir::Location loc) { if (mlir::isa(op)) + mlir::omp::AtomicUpdateOp, mlir::omp::LoopNestOp>(op)) return builder.create(loc); - else - return builder.create(loc); + return builder.create(loc); } void Fortran::lower::genOpenMPConstruct( diff --git a/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir b/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir index 92628af37085..fa7979e8875a 100644 --- a/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir +++ b/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir @@ -180,14 +180,16 @@ func.func @_QPsimd1(%arg0: !fir.ref {fir.bindc_name = "n"}, %arg1: !fir.ref omp.parallel { %1 = fir.alloca i32 {adapt.valuebyref, pinned} %2 = fir.load %arg0 : !fir.ref - omp.simdloop for (%arg2) : i32 = (%c1_i32) to (%2) step (%c1_i32) { - fir.store %arg2 to %1 : !fir.ref - %3 = fir.load %1 : !fir.ref - %4 = fir.convert %3 : (i32) -> i64 - %5 = arith.subi %4, %c1_i64 : i64 - %6 = fir.coordinate_of %arg1, %5 : (!fir.ref>, i64) -> !fir.ref - fir.store %3 to %6 : !fir.ref - omp.yield + omp.simd { + omp.loop_nest (%arg2) : i32 = (%c1_i32) to (%2) step (%c1_i32) { + fir.store %arg2 to %1 : !fir.ref + %3 = fir.load %1 : !fir.ref + %4 = fir.convert %3 : (i32) -> i64 + %5 = arith.subi %4, %c1_i64 : i64 + %6 = fir.coordinate_of %arg1, %5 : (!fir.ref>, i64) -> !fir.ref + fir.store %3 to %6 : !fir.ref + omp.yield + } } omp.terminator } @@ -202,8 +204,8 @@ func.func @_QPsimd1(%arg0: !fir.ref {fir.bindc_name = "n"}, %arg1: !fir.ref // CHECK: %[[ONE_3:.*]] = llvm.mlir.constant(1 : i64) : i64 // CHECK: %[[I_VAR:.*]] = llvm.alloca %[[ONE_3]] x i32 {pinned} : (i64) -> !llvm.ptr // CHECK: %[[N:.*]] = llvm.load %[[N_REF]] : !llvm.ptr -> i32 -// CHECK: omp.simdloop -// CHECK-SAME: (%[[I:.*]]) : i32 = (%[[ONE_2]]) to (%[[N]]) step (%[[ONE_2]]) { +// CHECK: omp.simd { +// CHECK-NEXT: omp.loop_nest (%[[I:.*]]) : i32 = (%[[ONE_2]]) to (%[[N]]) step (%[[ONE_2]]) { // CHECK: llvm.store %[[I]], %[[I_VAR]] : i32, !llvm.ptr // CHECK: %[[I1:.*]] = llvm.load %[[I_VAR]] : !llvm.ptr -> i32 // CHECK: %[[I1_EXT:.*]] = llvm.sext %[[I1]] : i32 to i64 @@ -212,6 +214,7 @@ func.func @_QPsimd1(%arg0: !fir.ref {fir.bindc_name = "n"}, %arg1: !fir.ref // CHECK: llvm.store %[[I1]], %[[ARR_I_REF]] : i32, !llvm.ptr // CHECK: omp.yield // CHECK: } +// CHECK: } // CHECK: omp.terminator // CHECK: } // CHECK: llvm.return @@ -471,55 +474,59 @@ func.func @_QPomp_target() { // ----- -func.func @_QPsimdloop_with_nested_loop() { +func.func @_QPsimd_with_nested_loop() { %0 = fir.alloca i32 {adapt.valuebyref} - %1 = fir.alloca !fir.array<10xi32> {bindc_name = "a", uniq_name = "_QFsimdloop_with_nested_loopEa"} - %2 = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimdloop_with_nested_loopEi"} - %3 = fir.alloca i32 {bindc_name = "j", uniq_name = "_QFsimdloop_with_nested_loopEj"} + %1 = fir.alloca !fir.array<10xi32> {bindc_name = "a", uniq_name = "_QFsimd_with_nested_loopEa"} + %2 = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimd_with_nested_loopEi"} + %3 = fir.alloca i32 {bindc_name = "j", uniq_name = "_QFsimd_with_nested_loopEj"} %c1_i32 = arith.constant 1 : i32 %c10_i32 = arith.constant 10 : i32 %c1_i32_0 = arith.constant 1 : i32 - omp.simdloop for (%arg0) : i32 = (%c1_i32) to (%c10_i32) inclusive step (%c1_i32_0) { - fir.store %arg0 to %0 : !fir.ref - %c1_i32_1 = arith.constant 1 : i32 - %4 = fir.convert %c1_i32_1 : (i32) -> index - %c10_i32_2 = arith.constant 10 : i32 - %5 = fir.convert %c10_i32_2 : (i32) -> index - %c1 = arith.constant 1 : index - %6 = fir.do_loop %arg1 = %4 to %5 step %c1 -> index { - %8 = fir.convert %arg1 : (index) -> i32 - fir.store %8 to %3 : !fir.ref - %9 = fir.load %0 : !fir.ref - %10 = fir.load %0 : !fir.ref - %11 = fir.convert %10 : (i32) -> i64 - %c1_i64 = arith.constant 1 : i64 - %12 = arith.subi %11, %c1_i64 : i64 - %13 = fir.coordinate_of %1, %12 : (!fir.ref>, i64) -> !fir.ref - fir.store %9 to %13 : !fir.ref - %14 = arith.addi %arg1, %c1 : index - fir.result %14 : index + omp.simd { + omp.loop_nest (%arg0) : i32 = (%c1_i32) to (%c10_i32) inclusive step (%c1_i32_0) { + fir.store %arg0 to %0 : !fir.ref + %c1_i32_1 = arith.constant 1 : i32 + %4 = fir.convert %c1_i32_1 : (i32) -> index + %c10_i32_2 = arith.constant 10 : i32 + %5 = fir.convert %c10_i32_2 : (i32) -> index + %c1 = arith.constant 1 : index + %6 = fir.do_loop %arg1 = %4 to %5 step %c1 -> index { + %8 = fir.convert %arg1 : (index) -> i32 + fir.store %8 to %3 : !fir.ref + %9 = fir.load %0 : !fir.ref + %10 = fir.load %0 : !fir.ref + %11 = fir.convert %10 : (i32) -> i64 + %c1_i64 = arith.constant 1 : i64 + %12 = arith.subi %11, %c1_i64 : i64 + %13 = fir.coordinate_of %1, %12 : (!fir.ref>, i64) -> !fir.ref + fir.store %9 to %13 : !fir.ref + %14 = arith.addi %arg1, %c1 : index + fir.result %14 : index + } + %7 = fir.convert %6 : (index) -> i32 + fir.store %7 to %3 : !fir.ref + omp.yield } - %7 = fir.convert %6 : (index) -> i32 - fir.store %7 to %3 : !fir.ref - omp.yield } return } -// CHECK-LABEL: llvm.func @_QPsimdloop_with_nested_loop() { +// CHECK-LABEL: llvm.func @_QPsimd_with_nested_loop() { // CHECK: %[[LOWER:.*]] = llvm.mlir.constant(1 : i32) : i32 // CHECK: %[[UPPER:.*]] = llvm.mlir.constant(10 : i32) : i32 // CHECK: %[[STEP:.*]] = llvm.mlir.constant(1 : i32) : i32 -// CHECK: omp.simdloop for (%[[CNT:.*]]) : i32 = (%[[LOWER]]) to (%[[UPPER]]) inclusive step (%[[STEP]]) { -// CHECK: llvm.br ^bb1(%[[VAL_1:.*]], %[[VAL_2:.*]] : i64, i64) -// CHECK: ^bb1(%[[VAL_3:.*]]: i64, %[[VAL_4:.*]]: i64): -// CHECK: %[[VAL_5:.*]] = llvm.mlir.constant(0 : index) : i64 -// CHECK: %[[VAL_6:.*]] = llvm.icmp "sgt" %[[VAL_4]], %[[VAL_5]] : i64 -// CHECK: llvm.cond_br %[[VAL_6]], ^bb2, ^bb3 -// CHECK: ^bb2: -// CHECK: llvm.br ^bb1(%[[VAL_7:.*]], %[[VAL_8:.*]] : i64, i64) -// CHECK: ^bb3: -// CHECK: omp.yield +// CHECK: omp.simd { +// CHECK-NEXT: omp.loop_nest (%[[CNT:.*]]) : i32 = (%[[LOWER]]) to (%[[UPPER]]) inclusive step (%[[STEP]]) { +// CHECK: llvm.br ^bb1(%[[VAL_1:.*]], %[[VAL_2:.*]] : i64, i64) +// CHECK: ^bb1(%[[VAL_3:.*]]: i64, %[[VAL_4:.*]]: i64): +// CHECK: %[[VAL_5:.*]] = llvm.mlir.constant(0 : index) : i64 +// CHECK: %[[VAL_6:.*]] = llvm.icmp "sgt" %[[VAL_4]], %[[VAL_5]] : i64 +// CHECK: llvm.cond_br %[[VAL_6]], ^bb2, ^bb3 +// CHECK: ^bb2: +// CHECK: llvm.br ^bb1(%[[VAL_7:.*]], %[[VAL_8:.*]] : i64, i64) +// CHECK: ^bb3: +// CHECK: omp.yield +// CHECK: } // CHECK: } // CHECK: llvm.return // CHECK: } diff --git a/flang/test/Lower/OpenMP/FIR/if-clause.f90 b/flang/test/Lower/OpenMP/FIR/if-clause.f90 index a1235be8e61e..f686b9708fc5 100644 --- a/flang/test/Lower/OpenMP/FIR/if-clause.f90 +++ b/flang/test/Lower/OpenMP/FIR/if-clause.f90 @@ -116,7 +116,7 @@ program main do i = 1, 10 end do !$omp end parallel do simd - + ! CHECK: omp.parallel ! CHECK-SAME: if({{.*}}) ! CHECK: omp.wsloop @@ -124,7 +124,7 @@ program main do i = 1, 10 end do !$omp end parallel do simd - + ! CHECK: omp.parallel ! CHECK-SAME: if({{.*}}) ! CHECK: omp.wsloop @@ -134,7 +134,7 @@ program main do i = 1, 10 end do !$omp end parallel do simd - + ! CHECK: omp.parallel ! CHECK-NOT: if({{.*}}) ! CHECK-SAME: { @@ -147,7 +147,7 @@ program main ! ---------------------------------------------------------------------------- ! SIMD ! ---------------------------------------------------------------------------- - ! CHECK: omp.simdloop + ! CHECK: omp.simd ! CHECK-NOT: if({{.*}}) ! CHECK-SAME: { !$omp simd @@ -155,14 +155,14 @@ program main end do !$omp end simd - ! CHECK: omp.simdloop + ! CHECK: omp.simd ! CHECK-SAME: if({{.*}}) !$omp simd if(.true.) do i = 1, 10 end do !$omp end simd - ! CHECK: omp.simdloop + ! CHECK: omp.simd ! CHECK-SAME: if({{.*}}) !$omp simd if(simd: .true.) do i = 1, 10 @@ -281,7 +281,6 @@ program main end do !$omp end target parallel do - ! CHECK: omp.target ! CHECK-NOT: if({{.*}}) ! CHECK-SAME: { @@ -360,7 +359,7 @@ program main ! CHECK: omp.target ! CHECK-NOT: if({{.*}}) ! CHECK-SAME: { - ! CHECK: omp.simdloop + ! CHECK: omp.simd ! CHECK-NOT: if({{.*}}) ! CHECK-SAME: { !$omp target simd @@ -370,7 +369,7 @@ program main ! CHECK: omp.target ! CHECK-SAME: if({{.*}}) - ! CHECK: omp.simdloop + ! CHECK: omp.simd ! CHECK-SAME: if({{.*}}) !$omp target simd if(.true.) do i = 1, 10 @@ -379,7 +378,7 @@ program main ! CHECK: omp.target ! CHECK-SAME: if({{.*}}) - ! CHECK: omp.simdloop + ! CHECK: omp.simd ! CHECK-SAME: if({{.*}}) !$omp target simd if(target: .true.) if(simd: .false.) do i = 1, 10 @@ -388,7 +387,7 @@ program main ! CHECK: omp.target ! CHECK-SAME: if({{.*}}) - ! CHECK: omp.simdloop + ! CHECK: omp.simd ! CHECK-NOT: if({{.*}}) ! CHECK-SAME: { !$omp target simd if(target: .true.) @@ -399,7 +398,7 @@ program main ! CHECK: omp.target ! CHECK-NOT: if({{.*}}) ! CHECK-SAME: { - ! CHECK: omp.simdloop + ! CHECK: omp.simd ! CHECK-SAME: if({{.*}}) !$omp target simd if(simd: .true.) do i = 1, 10 diff --git a/flang/test/Lower/OpenMP/FIR/loop-combined.f90 b/flang/test/Lower/OpenMP/FIR/loop-combined.f90 index a6cec1beb49c..6c6618dc9fb5 100644 --- a/flang/test/Lower/OpenMP/FIR/loop-combined.f90 +++ b/flang/test/Lower/OpenMP/FIR/loop-combined.f90 @@ -75,7 +75,7 @@ program main ! TARGET SIMD ! ---------------------------------------------------------------------------- ! CHECK: omp.target - ! CHECK: omp.simdloop + ! CHECK: omp.simd !$omp target simd do i = 1, 10 end do diff --git a/flang/test/Lower/OpenMP/FIR/parallel-private-clause.f90 b/flang/test/Lower/OpenMP/FIR/parallel-private-clause.f90 index 8f5d280943cc..8b75ecbaae8c 100644 --- a/flang/test/Lower/OpenMP/FIR/parallel-private-clause.f90 +++ b/flang/test/Lower/OpenMP/FIR/parallel-private-clause.f90 @@ -361,7 +361,8 @@ subroutine simd_loop_1 ! FIRDialect: %[[UB:.*]] = arith.constant 9 : i32 ! FIRDialect: %[[STEP:.*]] = arith.constant 1 : i32 - ! FIRDialect: omp.simdloop for (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { + ! FIRDialect: omp.simd { + ! FIRDialect-NEXT: omp.loop_nest (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { !$OMP SIMD PRIVATE(r) do i=1, 9 ! FIRDialect: fir.store %[[I]] to %[[LOCAL:.*]] : !fir.ref diff --git a/flang/test/Lower/OpenMP/FIR/simd.f90 b/flang/test/Lower/OpenMP/FIR/simd.f90 index c8c2022d693d..db7d30295c45 100644 --- a/flang/test/Lower/OpenMP/FIR/simd.f90 +++ b/flang/test/Lower/OpenMP/FIR/simd.f90 @@ -2,32 +2,34 @@ ! RUN: bbc -fopenmp -emit-fir -hlfir=false %s -o - | FileCheck %s -!CHECK-LABEL: func @_QPsimdloop() -subroutine simdloop -integer :: i +!CHECK-LABEL: func @_QPsimd() +subroutine simd + integer :: i !$OMP SIMD ! CHECK: %[[LB:.*]] = arith.constant 1 : i32 ! CHECK-NEXT: %[[UB:.*]] = arith.constant 9 : i32 ! CHECK-NEXT: %[[STEP:.*]] = arith.constant 1 : i32 - ! CHECK-NEXT: omp.simdloop for (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { + ! CHECK-NEXT: omp.simd { + ! CHECK-NEXT: omp.loop_nest (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { do i=1, 9 ! CHECK: fir.store %[[I]] to %[[LOCAL:.*]] : !fir.ref ! CHECK: %[[LD:.*]] = fir.load %[[LOCAL]] : !fir.ref ! CHECK: fir.call @_FortranAioOutputInteger32({{.*}}, %[[LD]]) {{.*}}: (!fir.ref, i32) -> i1 print*, i end do - !$OMP END SIMD + !$OMP END SIMD end subroutine -!CHECK-LABEL: func @_QPsimdloop_with_if_clause -subroutine simdloop_with_if_clause(n, threshold) -integer :: i, n, threshold +!CHECK-LABEL: func @_QPsimd_with_if_clause +subroutine simd_with_if_clause(n, threshold) + integer :: i, n, threshold !$OMP SIMD IF( n .GE. threshold ) ! CHECK: %[[LB:.*]] = arith.constant 1 : i32 ! CHECK: %[[UB:.*]] = fir.load %arg0 ! CHECK: %[[STEP:.*]] = arith.constant 1 : i32 ! CHECK: %[[COND:.*]] = arith.cmpi sge - ! CHECK: omp.simdloop if(%[[COND:.*]]) for (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { + ! CHECK: omp.simd if(%[[COND:.*]]) { + ! CHECK-NEXT: omp.loop_nest (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { do i = 1, n ! CHECK: fir.store %[[I]] to %[[LOCAL:.*]] : !fir.ref ! CHECK: %[[LD:.*]] = fir.load %[[LOCAL]] : !fir.ref @@ -37,14 +39,15 @@ integer :: i, n, threshold !$OMP END SIMD end subroutine -!CHECK-LABEL: func @_QPsimdloop_with_simdlen_clause -subroutine simdloop_with_simdlen_clause(n, threshold) -integer :: i, n, threshold +!CHECK-LABEL: func @_QPsimd_with_simdlen_clause +subroutine simd_with_simdlen_clause(n, threshold) + integer :: i, n, threshold !$OMP SIMD SIMDLEN(2) ! CHECK: %[[LB:.*]] = arith.constant 1 : i32 ! CHECK: %[[UB:.*]] = fir.load %arg0 ! CHECK: %[[STEP:.*]] = arith.constant 1 : i32 - ! CHECK: omp.simdloop simdlen(2) for (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { + ! CHECK: omp.simd simdlen(2) { + ! CHECK-NEXT: omp.loop_nest (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { do i = 1, n ! CHECK: fir.store %[[I]] to %[[LOCAL:.*]] : !fir.ref ! CHECK: %[[LD:.*]] = fir.load %[[LOCAL]] : !fir.ref @@ -54,15 +57,16 @@ integer :: i, n, threshold !$OMP END SIMD end subroutine -!CHECK-LABEL: func @_QPsimdloop_with_simdlen_clause_from_param -subroutine simdloop_with_simdlen_clause_from_param(n, threshold) -integer :: i, n, threshold -integer, parameter :: simdlen = 2; +!CHECK-LABEL: func @_QPsimd_with_simdlen_clause_from_param +subroutine simd_with_simdlen_clause_from_param(n, threshold) + integer :: i, n, threshold + integer, parameter :: simdlen = 2; !$OMP SIMD SIMDLEN(simdlen) ! CHECK: %[[LB:.*]] = arith.constant 1 : i32 ! CHECK: %[[UB:.*]] = fir.load %arg0 ! CHECK: %[[STEP:.*]] = arith.constant 1 : i32 - ! CHECK: omp.simdloop simdlen(2) for (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { + ! CHECK: omp.simd simdlen(2) { + ! CHECK-NEXT: omp.loop_nest (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { do i = 1, n ! CHECK: fir.store %[[I]] to %[[LOCAL:.*]] : !fir.ref ! CHECK: %[[LD:.*]] = fir.load %[[LOCAL]] : !fir.ref @@ -72,15 +76,16 @@ integer, parameter :: simdlen = 2; !$OMP END SIMD end subroutine -!CHECK-LABEL: func @_QPsimdloop_with_simdlen_clause_from_expr_from_param -subroutine simdloop_with_simdlen_clause_from_expr_from_param(n, threshold) -integer :: i, n, threshold -integer, parameter :: simdlen = 2; +!CHECK-LABEL: func @_QPsimd_with_simdlen_clause_from_expr_from_param +subroutine simd_with_simdlen_clause_from_expr_from_param(n, threshold) + integer :: i, n, threshold + integer, parameter :: simdlen = 2; !$OMP SIMD SIMDLEN(simdlen*2 + 2) ! CHECK: %[[LB:.*]] = arith.constant 1 : i32 ! CHECK: %[[UB:.*]] = fir.load %arg0 ! CHECK: %[[STEP:.*]] = arith.constant 1 : i32 - ! CHECK: omp.simdloop simdlen(6) for (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { + ! CHECK: omp.simd simdlen(6) { + ! CHECK-NEXT: omp.loop_nest (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { do i = 1, n ! CHECK: fir.store %[[I]] to %[[LOCAL:.*]] : !fir.ref ! CHECK: %[[LD:.*]] = fir.load %[[LOCAL]] : !fir.ref @@ -90,14 +95,15 @@ integer, parameter :: simdlen = 2; !$OMP END SIMD end subroutine -!CHECK-LABEL: func @_QPsimdloop_with_safelen_clause -subroutine simdloop_with_safelen_clause(n, threshold) -integer :: i, n, threshold +!CHECK-LABEL: func @_QPsimd_with_safelen_clause +subroutine simd_with_safelen_clause(n, threshold) + integer :: i, n, threshold !$OMP SIMD SAFELEN(2) ! CHECK: %[[LB:.*]] = arith.constant 1 : i32 ! CHECK: %[[UB:.*]] = fir.load %arg0 ! CHECK: %[[STEP:.*]] = arith.constant 1 : i32 - ! CHECK: omp.simdloop safelen(2) for (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { + ! CHECK: omp.simd safelen(2) { + ! CHECK-NEXT: omp.loop_nest (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { do i = 1, n ! CHECK: fir.store %[[I]] to %[[LOCAL:.*]] : !fir.ref ! CHECK: %[[LD:.*]] = fir.load %[[LOCAL]] : !fir.ref @@ -107,15 +113,16 @@ integer :: i, n, threshold !$OMP END SIMD end subroutine -!CHECK-LABEL: func @_QPsimdloop_with_safelen_clause_from_expr_from_param -subroutine simdloop_with_safelen_clause_from_expr_from_param(n, threshold) -integer :: i, n, threshold -integer, parameter :: safelen = 2; +!CHECK-LABEL: func @_QPsimd_with_safelen_clause_from_expr_from_param +subroutine simd_with_safelen_clause_from_expr_from_param(n, threshold) + integer :: i, n, threshold + integer, parameter :: safelen = 2; !$OMP SIMD SAFELEN(safelen*2 + 2) ! CHECK: %[[LB:.*]] = arith.constant 1 : i32 ! CHECK: %[[UB:.*]] = fir.load %arg0 ! CHECK: %[[STEP:.*]] = arith.constant 1 : i32 - ! CHECK: omp.simdloop safelen(6) for (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { + ! CHECK: omp.simd safelen(6) { + ! CHECK-NEXT: omp.loop_nest (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { do i = 1, n ! CHECK: fir.store %[[I]] to %[[LOCAL:.*]] : !fir.ref ! CHECK: %[[LD:.*]] = fir.load %[[LOCAL]] : !fir.ref @@ -125,14 +132,15 @@ integer, parameter :: safelen = 2; !$OMP END SIMD end subroutine -!CHECK-LABEL: func @_QPsimdloop_with_simdlen_safelen_clause -subroutine simdloop_with_simdlen_safelen_clause(n, threshold) -integer :: i, n, threshold +!CHECK-LABEL: func @_QPsimd_with_simdlen_safelen_clause +subroutine simd_with_simdlen_safelen_clause(n, threshold) + integer :: i, n, threshold !$OMP SIMD SIMDLEN(1) SAFELEN(2) ! CHECK: %[[LB:.*]] = arith.constant 1 : i32 ! CHECK: %[[UB:.*]] = fir.load %arg0 ! CHECK: %[[STEP:.*]] = arith.constant 1 : i32 - ! CHECK: omp.simdloop simdlen(1) safelen(2) for (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { + ! CHECK: omp.simd simdlen(1) safelen(2) { + ! CHECK-NEXT: omp.loop_nest (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { do i = 1, n ! CHECK: fir.store %[[I]] to %[[LOCAL:.*]] : !fir.ref ! CHECK: %[[LD:.*]] = fir.load %[[LOCAL]] : !fir.ref @@ -142,20 +150,21 @@ integer :: i, n, threshold !$OMP END SIMD end subroutine -!CHECK-LABEL: func @_QPsimdloop_with_collapse_clause -subroutine simdloop_with_collapse_clause(n) -integer :: i, j, n -integer :: A(n,n) -! CHECK: %[[LOWER_I:.*]] = arith.constant 1 : i32 -! CHECK: %[[UPPER_I:.*]] = fir.load %[[PARAM_ARG:.*]] : !fir.ref -! CHECK: %[[STEP_I:.*]] = arith.constant 1 : i32 -! CHECK: %[[LOWER_J:.*]] = arith.constant 1 : i32 -! CHECK: %[[UPPER_J:.*]] = fir.load %[[PARAM_ARG:.*]] : !fir.ref -! CHECK: %[[STEP_J:.*]] = arith.constant 1 : i32 -! CHECK: omp.simdloop for (%[[ARG_0:.*]], %[[ARG_1:.*]]) : i32 = ( -! CHECK-SAME: %[[LOWER_I]], %[[LOWER_J]]) to ( -! CHECK-SAME: %[[UPPER_I]], %[[UPPER_J]]) inclusive step ( -! CHECK-SAME: %[[STEP_I]], %[[STEP_J]]) { +!CHECK-LABEL: func @_QPsimd_with_collapse_clause +subroutine simd_with_collapse_clause(n) + integer :: i, j, n + integer :: A(n,n) + ! CHECK: %[[LOWER_I:.*]] = arith.constant 1 : i32 + ! CHECK: %[[UPPER_I:.*]] = fir.load %[[PARAM_ARG:.*]] : !fir.ref + ! CHECK: %[[STEP_I:.*]] = arith.constant 1 : i32 + ! CHECK: %[[LOWER_J:.*]] = arith.constant 1 : i32 + ! CHECK: %[[UPPER_J:.*]] = fir.load %[[PARAM_ARG:.*]] : !fir.ref + ! CHECK: %[[STEP_J:.*]] = arith.constant 1 : i32 + ! CHECK: omp.simd { + ! CHECK-NEXT: omp.loop_nest (%[[ARG_0:.*]], %[[ARG_1:.*]]) : i32 = ( + ! CHECK-SAME: %[[LOWER_I]], %[[LOWER_J]]) to ( + ! CHECK-SAME: %[[UPPER_I]], %[[UPPER_J]]) inclusive step ( + ! CHECK-SAME: %[[STEP_I]], %[[STEP_J]]) { !$OMP SIMD COLLAPSE(2) do i = 1, n do j = 1, n diff --git a/flang/test/Lower/OpenMP/if-clause.f90 b/flang/test/Lower/OpenMP/if-clause.f90 index f982bf67b072..ce4427a0c2ca 100644 --- a/flang/test/Lower/OpenMP/if-clause.f90 +++ b/flang/test/Lower/OpenMP/if-clause.f90 @@ -116,7 +116,7 @@ program main do i = 1, 10 end do !$omp end parallel do simd - + ! CHECK: omp.parallel ! CHECK-SAME: if({{.*}}) ! CHECK: omp.wsloop @@ -124,7 +124,7 @@ program main do i = 1, 10 end do !$omp end parallel do simd - + ! CHECK: omp.parallel ! CHECK-SAME: if({{.*}}) ! CHECK: omp.wsloop @@ -134,7 +134,7 @@ program main do i = 1, 10 end do !$omp end parallel do simd - + ! CHECK: omp.parallel ! CHECK-NOT: if({{.*}}) ! CHECK-SAME: { @@ -147,7 +147,7 @@ program main ! ---------------------------------------------------------------------------- ! SIMD ! ---------------------------------------------------------------------------- - ! CHECK: omp.simdloop + ! CHECK: omp.simd ! CHECK-NOT: if({{.*}}) ! CHECK-SAME: { !$omp simd @@ -155,14 +155,14 @@ program main end do !$omp end simd - ! CHECK: omp.simdloop + ! CHECK: omp.simd ! CHECK-SAME: if({{.*}}) !$omp simd if(.true.) do i = 1, 10 end do !$omp end simd - ! CHECK: omp.simdloop + ! CHECK: omp.simd ! CHECK-SAME: if({{.*}}) !$omp simd if(simd: .true.) do i = 1, 10 @@ -281,7 +281,6 @@ program main end do !$omp end target parallel do - ! CHECK: omp.target ! CHECK-NOT: if({{.*}}) ! CHECK-SAME: { @@ -360,7 +359,7 @@ program main ! CHECK: omp.target ! CHECK-NOT: if({{.*}}) ! CHECK-SAME: { - ! CHECK: omp.simdloop + ! CHECK: omp.simd ! CHECK-NOT: if({{.*}}) ! CHECK-SAME: { !$omp target simd @@ -370,7 +369,7 @@ program main ! CHECK: omp.target ! CHECK-SAME: if({{.*}}) - ! CHECK: omp.simdloop + ! CHECK: omp.simd ! CHECK-SAME: if({{.*}}) !$omp target simd if(.true.) do i = 1, 10 @@ -379,7 +378,7 @@ program main ! CHECK: omp.target ! CHECK-SAME: if({{.*}}) - ! CHECK: omp.simdloop + ! CHECK: omp.simd ! CHECK-SAME: if({{.*}}) !$omp target simd if(target: .true.) if(simd: .false.) do i = 1, 10 @@ -388,7 +387,7 @@ program main ! CHECK: omp.target ! CHECK-SAME: if({{.*}}) - ! CHECK: omp.simdloop + ! CHECK: omp.simd ! CHECK-NOT: if({{.*}}) ! CHECK-SAME: { !$omp target simd if(target: .true.) @@ -399,7 +398,7 @@ program main ! CHECK: omp.target ! CHECK-NOT: if({{.*}}) ! CHECK-SAME: { - ! CHECK: omp.simdloop + ! CHECK: omp.simd ! CHECK-SAME: if({{.*}}) !$omp target simd if(simd: .true.) do i = 1, 10 diff --git a/flang/test/Lower/OpenMP/loop-combined.f90 b/flang/test/Lower/OpenMP/loop-combined.f90 index 70488b6a769c..298634b3f6f8 100644 --- a/flang/test/Lower/OpenMP/loop-combined.f90 +++ b/flang/test/Lower/OpenMP/loop-combined.f90 @@ -75,7 +75,7 @@ program main ! TARGET SIMD ! ---------------------------------------------------------------------------- ! CHECK: omp.target - ! CHECK: omp.simdloop + ! CHECK: omp.simd !$omp target simd do i = 1, 10 end do diff --git a/flang/test/Lower/OpenMP/parallel-private-clause.f90 b/flang/test/Lower/OpenMP/parallel-private-clause.f90 index 5578b6710da7..775f7b4f2cb1 100644 --- a/flang/test/Lower/OpenMP/parallel-private-clause.f90 +++ b/flang/test/Lower/OpenMP/parallel-private-clause.f90 @@ -411,7 +411,8 @@ subroutine simd_loop_1 ! FIRDialect: %[[UB:.*]] = arith.constant 9 : i32 ! FIRDialect: %[[STEP:.*]] = arith.constant 1 : i32 - ! FIRDialect: omp.simdloop for (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { + ! FIRDialect: omp.simd { + ! FIRDialect-NEXT: omp.loop_nest (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { !$OMP SIMD PRIVATE(r) do i=1, 9 ! FIRDialect: fir.store %[[I]] to %[[LOCAL:.*]]#1 : !fir.ref diff --git a/flang/test/Lower/OpenMP/simd.f90 b/flang/test/Lower/OpenMP/simd.f90 index 135b38c79262..190aa6152121 100644 --- a/flang/test/Lower/OpenMP/simd.f90 +++ b/flang/test/Lower/OpenMP/simd.f90 @@ -3,33 +3,35 @@ !RUN: %flang_fc1 -flang-experimental-hlfir -emit-hlfir -fopenmp %s -o - | FileCheck %s !RUN: bbc -hlfir -emit-hlfir -fopenmp %s -o - | FileCheck %s -!CHECK-LABEL: func @_QPsimdloop() -subroutine simdloop -integer :: i +!CHECK-LABEL: func @_QPsimd() +subroutine simd + integer :: i !$OMP SIMD ! CHECK: %[[LB:.*]] = arith.constant 1 : i32 ! CHECK-NEXT: %[[UB:.*]] = arith.constant 9 : i32 ! CHECK-NEXT: %[[STEP:.*]] = arith.constant 1 : i32 - ! CHECK-NEXT: omp.simdloop for (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { + ! CHECK-NEXT: omp.simd { + ! CHECK-NEXT: omp.loop_nest (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { do i=1, 9 ! CHECK: fir.store %[[I]] to %[[LOCAL:.*]]#1 : !fir.ref ! CHECK: %[[LD:.*]] = fir.load %[[LOCAL]]#0 : !fir.ref ! CHECK: fir.call @_FortranAioOutputInteger32({{.*}}, %[[LD]]) {{.*}}: (!fir.ref, i32) -> i1 print*, i end do - !$OMP END SIMD + !$OMP END SIMD end subroutine -!CHECK-LABEL: func @_QPsimdloop_with_if_clause -subroutine simdloop_with_if_clause(n, threshold) - ! CHECK: %[[ARG_N:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFsimdloop_with_if_clauseEn"} : (!fir.ref) -> (!fir.ref, !fir.ref) -integer :: i, n, threshold +!CHECK-LABEL: func @_QPsimd_with_if_clause +subroutine simd_with_if_clause(n, threshold) + ! CHECK: %[[ARG_N:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFsimd_with_if_clauseEn"} : (!fir.ref) -> (!fir.ref, !fir.ref) + integer :: i, n, threshold !$OMP SIMD IF( n .GE. threshold ) ! CHECK: %[[LB:.*]] = arith.constant 1 : i32 ! CHECK: %[[UB:.*]] = fir.load %[[ARG_N]]#0 ! CHECK: %[[STEP:.*]] = arith.constant 1 : i32 ! CHECK: %[[COND:.*]] = arith.cmpi sge - ! CHECK: omp.simdloop if(%[[COND:.*]]) for (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { + ! CHECK: omp.simd if(%[[COND:.*]]) { + ! CHECK-NEXT: omp.loop_nest (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { do i = 1, n ! CHECK: fir.store %[[I]] to %[[LOCAL:.*]]#1 : !fir.ref ! CHECK: %[[LD:.*]] = fir.load %[[LOCAL]]#0 : !fir.ref @@ -39,15 +41,16 @@ integer :: i, n, threshold !$OMP END SIMD end subroutine -!CHECK-LABEL: func @_QPsimdloop_with_simdlen_clause -subroutine simdloop_with_simdlen_clause(n, threshold) - ! CHECK: %[[ARG_N:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFsimdloop_with_simdlen_clauseEn"} : (!fir.ref) -> (!fir.ref, !fir.ref) -integer :: i, n, threshold +!CHECK-LABEL: func @_QPsimd_with_simdlen_clause +subroutine simd_with_simdlen_clause(n, threshold) + ! CHECK: %[[ARG_N:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFsimd_with_simdlen_clauseEn"} : (!fir.ref) -> (!fir.ref, !fir.ref) + integer :: i, n, threshold !$OMP SIMD SIMDLEN(2) ! CHECK: %[[LB:.*]] = arith.constant 1 : i32 ! CHECK: %[[UB:.*]] = fir.load %[[ARG_N]]#0 ! CHECK: %[[STEP:.*]] = arith.constant 1 : i32 - ! CHECK: omp.simdloop simdlen(2) for (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { + ! CHECK: omp.simd simdlen(2) { + ! CHECK-NEXT: omp.loop_nest (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { do i = 1, n ! CHECK: fir.store %[[I]] to %[[LOCAL:.*]]#1 : !fir.ref ! CHECK: %[[LD:.*]] = fir.load %[[LOCAL]]#0 : !fir.ref @@ -57,16 +60,17 @@ integer :: i, n, threshold !$OMP END SIMD end subroutine -!CHECK-LABEL: func @_QPsimdloop_with_simdlen_clause_from_param -subroutine simdloop_with_simdlen_clause_from_param(n, threshold) - ! CHECK: %[[ARG_N:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFsimdloop_with_simdlen_clause_from_paramEn"} : (!fir.ref) -> (!fir.ref, !fir.ref) -integer :: i, n, threshold -integer, parameter :: simdlen = 2; +!CHECK-LABEL: func @_QPsimd_with_simdlen_clause_from_param +subroutine simd_with_simdlen_clause_from_param(n, threshold) + ! CHECK: %[[ARG_N:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFsimd_with_simdlen_clause_from_paramEn"} : (!fir.ref) -> (!fir.ref, !fir.ref) + integer :: i, n, threshold + integer, parameter :: simdlen = 2; !$OMP SIMD SIMDLEN(simdlen) ! CHECK: %[[LB:.*]] = arith.constant 1 : i32 ! CHECK: %[[UB:.*]] = fir.load %[[ARG_N]]#0 ! CHECK: %[[STEP:.*]] = arith.constant 1 : i32 - ! CHECK: omp.simdloop simdlen(2) for (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { + ! CHECK: omp.simd simdlen(2) { + ! CHECK-NEXT: omp.loop_nest (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { do i = 1, n ! CHECK: fir.store %[[I]] to %[[LOCAL:.*]]#1 : !fir.ref ! CHECK: %[[LD:.*]] = fir.load %[[LOCAL]]#0 : !fir.ref @@ -76,16 +80,17 @@ integer, parameter :: simdlen = 2; !$OMP END SIMD end subroutine -!CHECK-LABEL: func @_QPsimdloop_with_simdlen_clause_from_expr_from_param -subroutine simdloop_with_simdlen_clause_from_expr_from_param(n, threshold) - ! CHECK: %[[ARG_N:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFsimdloop_with_simdlen_clause_from_expr_from_paramEn"} : (!fir.ref) -> (!fir.ref, !fir.ref) -integer :: i, n, threshold -integer, parameter :: simdlen = 2; +!CHECK-LABEL: func @_QPsimd_with_simdlen_clause_from_expr_from_param +subroutine simd_with_simdlen_clause_from_expr_from_param(n, threshold) + ! CHECK: %[[ARG_N:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFsimd_with_simdlen_clause_from_expr_from_paramEn"} : (!fir.ref) -> (!fir.ref, !fir.ref) + integer :: i, n, threshold + integer, parameter :: simdlen = 2; !$OMP SIMD SIMDLEN(simdlen*2 + 2) ! CHECK: %[[LB:.*]] = arith.constant 1 : i32 ! CHECK: %[[UB:.*]] = fir.load %[[ARG_N]]#0 ! CHECK: %[[STEP:.*]] = arith.constant 1 : i32 - ! CHECK: omp.simdloop simdlen(6) for (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { + ! CHECK: omp.simd simdlen(6) { + ! CHECK-NEXT: omp.loop_nest (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { do i = 1, n ! CHECK: fir.store %[[I]] to %[[LOCAL:.*]]#1 : !fir.ref ! CHECK: %[[LD:.*]] = fir.load %[[LOCAL]]#0 : !fir.ref @@ -95,15 +100,16 @@ integer, parameter :: simdlen = 2; !$OMP END SIMD end subroutine -!CHECK-LABEL: func @_QPsimdloop_with_safelen_clause -subroutine simdloop_with_safelen_clause(n, threshold) - ! CHECK: %[[ARG_N:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFsimdloop_with_safelen_clauseEn"} : (!fir.ref) -> (!fir.ref, !fir.ref) -integer :: i, n, threshold +!CHECK-LABEL: func @_QPsimd_with_safelen_clause +subroutine simd_with_safelen_clause(n, threshold) + ! CHECK: %[[ARG_N:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFsimd_with_safelen_clauseEn"} : (!fir.ref) -> (!fir.ref, !fir.ref) + integer :: i, n, threshold !$OMP SIMD SAFELEN(2) ! CHECK: %[[LB:.*]] = arith.constant 1 : i32 ! CHECK: %[[UB:.*]] = fir.load %[[ARG_N]]#0 ! CHECK: %[[STEP:.*]] = arith.constant 1 : i32 - ! CHECK: omp.simdloop safelen(2) for (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { + ! CHECK: omp.simd safelen(2) { + ! CHECK-NEXT: omp.loop_nest (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { do i = 1, n ! CHECK: fir.store %[[I]] to %[[LOCAL:.*]]#1 : !fir.ref ! CHECK: %[[LD:.*]] = fir.load %[[LOCAL]]#0 : !fir.ref @@ -113,16 +119,17 @@ integer :: i, n, threshold !$OMP END SIMD end subroutine -!CHECK-LABEL: func @_QPsimdloop_with_safelen_clause_from_expr_from_param -subroutine simdloop_with_safelen_clause_from_expr_from_param(n, threshold) - ! CHECK: %[[ARG_N:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFsimdloop_with_safelen_clause_from_expr_from_paramEn"} : (!fir.ref) -> (!fir.ref, !fir.ref) -integer :: i, n, threshold -integer, parameter :: safelen = 2; +!CHECK-LABEL: func @_QPsimd_with_safelen_clause_from_expr_from_param +subroutine simd_with_safelen_clause_from_expr_from_param(n, threshold) + ! CHECK: %[[ARG_N:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFsimd_with_safelen_clause_from_expr_from_paramEn"} : (!fir.ref) -> (!fir.ref, !fir.ref) + integer :: i, n, threshold + integer, parameter :: safelen = 2; !$OMP SIMD SAFELEN(safelen*2 + 2) ! CHECK: %[[LB:.*]] = arith.constant 1 : i32 ! CHECK: %[[UB:.*]] = fir.load %[[ARG_N]]#0 ! CHECK: %[[STEP:.*]] = arith.constant 1 : i32 - ! CHECK: omp.simdloop safelen(6) for (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { + ! CHECK: omp.simd safelen(6) { + ! CHECK-NEXT: omp.loop_nest (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { do i = 1, n ! CHECK: fir.store %[[I]] to %[[LOCAL:.*]]#1 : !fir.ref ! CHECK: %[[LD:.*]] = fir.load %[[LOCAL]]#0 : !fir.ref @@ -132,15 +139,16 @@ integer, parameter :: safelen = 2; !$OMP END SIMD end subroutine -!CHECK-LABEL: func @_QPsimdloop_with_simdlen_safelen_clause -subroutine simdloop_with_simdlen_safelen_clause(n, threshold) - ! CHECK: %[[ARG_N:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFsimdloop_with_simdlen_safelen_clauseEn"} : (!fir.ref) -> (!fir.ref, !fir.ref) -integer :: i, n, threshold +!CHECK-LABEL: func @_QPsimd_with_simdlen_safelen_clause +subroutine simd_with_simdlen_safelen_clause(n, threshold) + ! CHECK: %[[ARG_N:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFsimd_with_simdlen_safelen_clauseEn"} : (!fir.ref) -> (!fir.ref, !fir.ref) + integer :: i, n, threshold !$OMP SIMD SIMDLEN(1) SAFELEN(2) ! CHECK: %[[LB:.*]] = arith.constant 1 : i32 ! CHECK: %[[UB:.*]] = fir.load %[[ARG_N]]#0 ! CHECK: %[[STEP:.*]] = arith.constant 1 : i32 - ! CHECK: omp.simdloop simdlen(1) safelen(2) for (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { + ! CHECK: omp.simd simdlen(1) safelen(2) { + ! CHECK-NEXT: omp.loop_nest (%[[I:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) { do i = 1, n ! CHECK: fir.store %[[I]] to %[[LOCAL:.*]]#1 : !fir.ref ! CHECK: %[[LD:.*]] = fir.load %[[LOCAL]]#0 : !fir.ref @@ -150,20 +158,21 @@ integer :: i, n, threshold !$OMP END SIMD end subroutine -!CHECK-LABEL: func @_QPsimdloop_with_collapse_clause -subroutine simdloop_with_collapse_clause(n) -integer :: i, j, n -integer :: A(n,n) -! CHECK: %[[LOWER_I:.*]] = arith.constant 1 : i32 -! CHECK: %[[UPPER_I:.*]] = fir.load %[[PARAM_ARG:.*]] : !fir.ref -! CHECK: %[[STEP_I:.*]] = arith.constant 1 : i32 -! CHECK: %[[LOWER_J:.*]] = arith.constant 1 : i32 -! CHECK: %[[UPPER_J:.*]] = fir.load %[[PARAM_ARG:.*]] : !fir.ref -! CHECK: %[[STEP_J:.*]] = arith.constant 1 : i32 -! CHECK: omp.simdloop for (%[[ARG_0:.*]], %[[ARG_1:.*]]) : i32 = ( -! CHECK-SAME: %[[LOWER_I]], %[[LOWER_J]]) to ( -! CHECK-SAME: %[[UPPER_I]], %[[UPPER_J]]) inclusive step ( -! CHECK-SAME: %[[STEP_I]], %[[STEP_J]]) { +!CHECK-LABEL: func @_QPsimd_with_collapse_clause +subroutine simd_with_collapse_clause(n) + integer :: i, j, n + integer :: A(n,n) + ! CHECK: %[[LOWER_I:.*]] = arith.constant 1 : i32 + ! CHECK: %[[UPPER_I:.*]] = fir.load %[[PARAM_ARG:.*]] : !fir.ref + ! CHECK: %[[STEP_I:.*]] = arith.constant 1 : i32 + ! CHECK: %[[LOWER_J:.*]] = arith.constant 1 : i32 + ! CHECK: %[[UPPER_J:.*]] = fir.load %[[PARAM_ARG:.*]] : !fir.ref + ! CHECK: %[[STEP_J:.*]] = arith.constant 1 : i32 + ! CHECK: omp.simd { + ! CHECK-NEXT: omp.loop_nest (%[[ARG_0:.*]], %[[ARG_1:.*]]) : i32 = ( + ! CHECK-SAME: %[[LOWER_I]], %[[LOWER_J]]) to ( + ! CHECK-SAME: %[[UPPER_I]], %[[UPPER_J]]) inclusive step ( + ! CHECK-SAME: %[[STEP_I]], %[[STEP_J]]) { !$OMP SIMD COLLAPSE(2) do i = 1, n do j = 1, n diff --git a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp index db1c4a8951ad..8344bca08404 100644 --- a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp +++ b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp @@ -2097,7 +2097,7 @@ TEST_F(OpenMPIRBuilderTest, ApplySimdlenSafelen) { })); } -TEST_F(OpenMPIRBuilderTest, ApplySimdLoopIf) { +TEST_F(OpenMPIRBuilderTest, ApplySimdIf) { OpenMPIRBuilder OMPBuilder(*M); IRBuilder<> Builder(BB); MapVector AlignedVars; diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPClauseOperands.h b/mlir/include/mlir/Dialect/OpenMP/OpenMPClauseOperands.h index 27a766aceb31..3c5fa23bd4a7 100644 --- a/mlir/include/mlir/Dialect/OpenMP/OpenMPClauseOperands.h +++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPClauseOperands.h @@ -251,11 +251,10 @@ using SectionsClauseOps = detail::Clauses; // TODO `linear` clause. -using SimdLoopClauseOps = - detail::Clauses; +using SimdClauseOps = + detail::Clauses; using SingleClauseOps = detail::Clauses; diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td index 82be7ad31a15..10771f6e854d 100644 --- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td +++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td @@ -562,7 +562,7 @@ def LoopNestOp : OpenMP_Op<"loop_nest", [SameVariadicOperandSize, loop operations intended to serve as a stopgap solution until the long-term representation of canonical loops is defined. Specifically, this operation is intended to serve as a unique source for loop information during the - transition to making `omp.distribute`, `omp.simdloop`, `omp.taskloop` and + transition to making `omp.distribute`, `omp.simd`, `omp.taskloop` and `omp.wsloop` wrapper operations. It is not intended to help with the addition of support for loop transformations, non-rectangular loops and non-perfectly nested loops. @@ -722,24 +722,19 @@ def WsloopOp : OpenMP_Op<"wsloop", [AttrSizedOperandSegments, // Simd construct [2.9.3.1] //===----------------------------------------------------------------------===// -def SimdLoopOp : OpenMP_Op<"simdloop", [AttrSizedOperandSegments, - AllTypesMatch<["lowerBound", "upperBound", "step"]>, - DeclareOpInterfaceMethods, - RecursiveMemoryEffects]> { - let summary = "simd loop construct"; +def SimdOp : OpenMP_Op<"simd", [AttrSizedOperandSegments, + DeclareOpInterfaceMethods, + RecursiveMemoryEffects, + SingleBlockImplicitTerminator<"TerminatorOp">]> { + let summary = "simd construct"; let description = [{ The simd construct can be applied to a loop to indicate that the loop can be transformed into a SIMD loop (that is, multiple iterations of the loop can - be executed concurrently using SIMD instructions).. The lower and upper - bounds specify a half-open range: the range includes the lower bound but - does not include the upper bound. If the `inclusive` attribute is specified - then the upper bound is also included. + be executed concurrently using SIMD instructions). - The body region can contain any number of blocks. The region is terminated - by "omp.yield" instruction without operands. - - Collapsed loops are represented by the simd-loop having a list of indices, - bounds and steps where the size of the list is equal to the collapse value. + The body region can contain a single block which must contain a single + operation and a terminator. The operation must be another compatible loop + wrapper or an `omp.loop_nest`. The `alignment_values` attribute additionally specifies alignment of each corresponding aligned operand. Note that `$aligned_vars` and @@ -763,32 +758,32 @@ def SimdLoopOp : OpenMP_Op<"simdloop", [AttrSizedOperandSegments, SIMD chunk can have a distance in the logical iteration space that is greater than or equal to the value given in the clause. ``` - omp.simdloop - for (%i1, %i2) : index = (%c0, %c0) to (%c10, %c10) step (%c1, %c1) { - // block operations - omp.yield + omp.simd { + omp.loop_nest (%i1, %i2) : index = (%c0, %c0) to (%c10, %c10) step (%c1, %c1) { + %a = load %arrA[%i1, %i2] : memref + %b = load %arrB[%i1, %i2] : memref + %sum = arith.addf %a, %b : f32 + store %sum, %arrC[%i1, %i2] : memref + omp.yield + } } ``` }]; // TODO: Add other clauses - let arguments = (ins Variadic:$lowerBound, - Variadic:$upperBound, - Variadic:$step, - Variadic:$aligned_vars, + let arguments = (ins Variadic:$aligned_vars, OptionalAttr:$alignment_values, Optional:$if_expr, Variadic:$nontemporal_vars, OptionalAttr:$order_val, ConfinedAttr, [IntPositive]>:$simdlen, - ConfinedAttr, [IntPositive]>:$safelen, - UnitAttr:$inclusive + ConfinedAttr, [IntPositive]>:$safelen ); let regions = (region AnyRegion:$region); let builders = [ - OpBuilder<(ins CArg<"const SimdLoopClauseOps &">:$clauses)> + OpBuilder<(ins CArg<"const SimdClauseOps &">:$clauses)> ]; let assemblyFormat = [{ @@ -800,14 +795,7 @@ def SimdLoopOp : OpenMP_Op<"simdloop", [AttrSizedOperandSegments, |`order` `(` custom($order_val) `)` |`simdlen` `(` $simdlen `)` |`safelen` `(` $safelen `)` - ) `for` custom($region, $lowerBound, $upperBound, $step, - type($step), $inclusive) attr-dict - }]; - - let extraClassDeclaration = [{ - /// Returns the number of loops in the simd loop nest. - unsigned getNumLoops() { return getLowerBound().size(); } - + ) $region attr-dict }]; let hasCustomAssemblyFormat = 1; @@ -818,7 +806,7 @@ def SimdLoopOp : OpenMP_Op<"simdloop", [AttrSizedOperandSegments, def YieldOp : OpenMP_Op<"yield", [Pure, ReturnLike, Terminator, ParentOneOf<["LoopNestOp", "WsloopOp", "DeclareReductionOp", - "AtomicUpdateOp", "SimdLoopOp", "PrivateClauseOp"]>]> { + "AtomicUpdateOp", "PrivateClauseOp"]>]> { let summary = "loop yield and termination operation"; let description = [{ "omp.yield" yields SSA values from the OpenMP dialect op region and diff --git a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp index b9ada0fa0f97..a206c7b228d2 100644 --- a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp +++ b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp @@ -251,11 +251,11 @@ void mlir::configureOpenMPToLLVMConversionLegality( }); target.addDynamicallyLegalOp< mlir::omp::AtomicUpdateOp, mlir::omp::CriticalOp, mlir::omp::TargetOp, - mlir::omp::TargetDataOp, mlir::omp::OrderedRegionOp, - mlir::omp::ParallelOp, mlir::omp::WsloopOp, mlir::omp::SimdLoopOp, - mlir::omp::MasterOp, mlir::omp::SectionOp, mlir::omp::SectionsOp, - mlir::omp::SingleOp, mlir::omp::TaskgroupOp, mlir::omp::TaskOp, - mlir::omp::DeclareReductionOp, + mlir::omp::TargetDataOp, mlir::omp::LoopNestOp, + mlir::omp::OrderedRegionOp, mlir::omp::ParallelOp, mlir::omp::WsloopOp, + mlir::omp::SimdOp, mlir::omp::MasterOp, mlir::omp::SectionOp, + mlir::omp::SectionsOp, mlir::omp::SingleOp, mlir::omp::TaskgroupOp, + mlir::omp::TaskOp, mlir::omp::DeclareReductionOp, mlir::omp::PrivateClauseOp>([&](Operation *op) { return std::all_of(op->getRegions().begin(), op->getRegions().end(), [&](Region ®ion) { @@ -278,11 +278,12 @@ void mlir::populateOpenMPToLLVMConversionPatterns(LLVMTypeConverter &converter, AtomicReadOpConversion, MapInfoOpConversion, ReductionOpConversion, MultiRegionOpConversion, MultiRegionOpConversion, - RegionOpConversion, RegionOpConversion, - ReductionOpConversion, RegionOpConversion, + RegionOpConversion, RegionOpConversion, + RegionOpConversion, ReductionOpConversion, + RegionOpConversion, RegionOpConversion, RegionOpConversion, RegionOpConversion, RegionOpConversion, - RegionOpConversion, RegionOpConversion, + RegionOpConversion, RegionOpConversion, RegionOpConversion, RegionOpConversion, RegionOpConversion, RegionOpConversion, RegionLessOpWithVarOperandsConversion, diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp index e500d0fca741..caf0ac3f8601 100644 --- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp +++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp @@ -1604,34 +1604,37 @@ void printLoopControl(OpAsmPrinter &p, Operation *op, Region ®ion, // Simd construct [2.9.3.1] //===----------------------------------------------------------------------===// -void SimdLoopOp::build(OpBuilder &builder, OperationState &state, - const SimdLoopClauseOps &clauses) { +void SimdOp::build(OpBuilder &builder, OperationState &state, + const SimdClauseOps &clauses) { MLIRContext *ctx = builder.getContext(); // TODO Store clauses in op: privateVars, reductionByRefAttr, reductionVars, // privatizers, reductionDeclSymbols. - SimdLoopOp::build( - builder, state, clauses.loopLBVar, clauses.loopUBVar, clauses.loopStepVar, - clauses.alignedVars, makeArrayAttr(ctx, clauses.alignmentAttrs), - clauses.ifVar, clauses.nontemporalVars, clauses.orderAttr, - clauses.simdlenAttr, clauses.safelenAttr, clauses.loopInclusiveAttr); + SimdOp::build(builder, state, clauses.alignedVars, + makeArrayAttr(ctx, clauses.alignmentAttrs), clauses.ifVar, + clauses.nontemporalVars, clauses.orderAttr, clauses.simdlenAttr, + clauses.safelenAttr); } -LogicalResult SimdLoopOp::verify() { - if (this->getLowerBound().empty()) { - return emitOpError() << "empty lowerbound for simd loop operation"; - } - if (this->getSimdlen().has_value() && this->getSafelen().has_value() && - this->getSimdlen().value() > this->getSafelen().value()) { +LogicalResult SimdOp::verify() { + if (getSimdlen().has_value() && getSafelen().has_value() && + getSimdlen().value() > getSafelen().value()) return emitOpError() << "simdlen clause and safelen clause are both present, but the " "simdlen value is not less than or equal to safelen value"; - } - if (verifyAlignedClause(*this, this->getAlignmentValues(), - this->getAlignedVars()) + + if (verifyAlignedClause(*this, getAlignmentValues(), getAlignedVars()) .failed()) return failure(); - if (verifyNontemporalClause(*this, this->getNontemporalVars()).failed()) + + if (verifyNontemporalClause(*this, getNontemporalVars()).failed()) return failure(); + + if (!isWrapper()) + return emitOpError() << "must be a loop wrapper"; + + if (getNestedWrapper()) + return emitOpError() << "must wrap an 'omp.loop_nest' directly"; + return success(); } @@ -1662,9 +1665,9 @@ LogicalResult DistributeOp::verify() { if (LoopWrapperInterface nested = getNestedWrapper()) { // Check for the allowed leaf constructs that may appear in a composite // construct directly after DISTRIBUTE. - if (!isa(nested)) + if (!isa(nested)) return emitError() << "only supported nested wrappers are 'omp.parallel' " - "and 'omp.simdloop'"; + "and 'omp.simd'"; } return success(); @@ -1876,8 +1879,8 @@ LogicalResult TaskloopOp::verify() { if (LoopWrapperInterface nested = getNestedWrapper()) { // Check for the allowed leaf constructs that may appear in a composite // construct directly after TASKLOOP. - if (!isa(nested)) - return emitError() << "only supported nested wrapper is 'omp.simdloop'"; + if (!isa(nested)) + return emitError() << "only supported nested wrapper is 'omp.simd'"; } return success(); } diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp index 300fc8ba56fc..e89ff9209b03 100644 --- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp @@ -1406,9 +1406,10 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder, /// Converts an OpenMP simd loop into LLVM IR using OpenMPIRBuilder. static LogicalResult -convertOmpSimdLoop(Operation &opInst, llvm::IRBuilderBase &builder, - LLVM::ModuleTranslation &moduleTranslation) { - auto loop = cast(opInst); +convertOmpSimd(Operation &opInst, llvm::IRBuilderBase &builder, + LLVM::ModuleTranslation &moduleTranslation) { + auto simdOp = cast(opInst); + auto loopOp = cast(simdOp.getWrappedLoop()); llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); @@ -1421,33 +1422,34 @@ convertOmpSimdLoop(Operation &opInst, llvm::IRBuilderBase &builder, auto bodyGen = [&](llvm::OpenMPIRBuilder::InsertPointTy ip, llvm::Value *iv) { // Make sure further conversions know about the induction variable. moduleTranslation.mapValue( - loop.getRegion().front().getArgument(loopInfos.size()), iv); + loopOp.getRegion().front().getArgument(loopInfos.size()), iv); // Capture the body insertion point for use in nested loops. BodyIP of the // CanonicalLoopInfo always points to the beginning of the entry block of // the body. bodyInsertPoints.push_back(ip); - if (loopInfos.size() != loop.getNumLoops() - 1) + if (loopInfos.size() != loopOp.getNumLoops() - 1) return; // Convert the body of the loop. builder.restoreIP(ip); - convertOmpOpRegions(loop.getRegion(), "omp.simdloop.region", builder, + convertOmpOpRegions(loopOp.getRegion(), "omp.simd.region", builder, moduleTranslation, bodyGenStatus); }; // Delegate actual loop construction to the OpenMP IRBuilder. - // TODO: this currently assumes SimdLoop is semantically similar to SCF loop, - // i.e. it has a positive step, uses signed integer semantics. Reconsider - // this code when SimdLoop clearly supports more cases. + // TODO: this currently assumes omp.loop_nest is semantically similar to SCF + // loop, i.e. it has a positive step, uses signed integer semantics. + // Reconsider this code when the nested loop operation clearly supports more + // cases. llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder(); - for (unsigned i = 0, e = loop.getNumLoops(); i < e; ++i) { + for (unsigned i = 0, e = loopOp.getNumLoops(); i < e; ++i) { llvm::Value *lowerBound = - moduleTranslation.lookupValue(loop.getLowerBound()[i]); + moduleTranslation.lookupValue(loopOp.getLowerBound()[i]); llvm::Value *upperBound = - moduleTranslation.lookupValue(loop.getUpperBound()[i]); - llvm::Value *step = moduleTranslation.lookupValue(loop.getStep()[i]); + moduleTranslation.lookupValue(loopOp.getUpperBound()[i]); + llvm::Value *step = moduleTranslation.lookupValue(loopOp.getStep()[i]); // Make sure loop trip count are emitted in the preheader of the outermost // loop at the latest so that they are all available for the new collapsed @@ -1473,18 +1475,18 @@ convertOmpSimdLoop(Operation &opInst, llvm::IRBuilderBase &builder, ompBuilder->collapseLoops(ompLoc.DL, loopInfos, {}); llvm::ConstantInt *simdlen = nullptr; - if (std::optional simdlenVar = loop.getSimdlen()) + if (std::optional simdlenVar = simdOp.getSimdlen()) simdlen = builder.getInt64(simdlenVar.value()); llvm::ConstantInt *safelen = nullptr; - if (std::optional safelenVar = loop.getSafelen()) + if (std::optional safelenVar = simdOp.getSafelen()) safelen = builder.getInt64(safelenVar.value()); llvm::MapVector alignedVars; ompBuilder->applySimd( loopInfo, alignedVars, - loop.getIfExpr() ? moduleTranslation.lookupValue(loop.getIfExpr()) - : nullptr, + simdOp.getIfExpr() ? moduleTranslation.lookupValue(simdOp.getIfExpr()) + : nullptr, llvm::omp::OrderKind::OMP_ORDER_unknown, simdlen, safelen); builder.restoreIP(afterIP); @@ -3198,8 +3200,8 @@ convertHostOrTargetOperation(Operation *op, llvm::IRBuilderBase &builder, .Case([&](omp::WsloopOp) { return convertOmpWsloop(*op, builder, moduleTranslation); }) - .Case([&](omp::SimdLoopOp) { - return convertOmpSimdLoop(*op, builder, moduleTranslation); + .Case([&](omp::SimdOp) { + return convertOmpSimd(*op, builder, moduleTranslation); }) .Case([&](omp::AtomicReadOp) { return convertOmpAtomicRead(*op, builder, moduleTranslation); @@ -3421,7 +3423,6 @@ LogicalResult OpenMPDialectLLVMIRTranslationInterface::convertOperation( return convertTargetOpsInNest(op, builder, moduleTranslation); } } - return convertHostOrTargetOperation(op, builder, moduleTranslation); } diff --git a/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir b/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir index dc5d6969ca78..9f45d139b81f 100644 --- a/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir +++ b/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir @@ -145,9 +145,10 @@ func.func @threadprivate(%a: !llvm.ptr) -> () { // ----- -// CHECK: llvm.func @simdloop_block_arg(%[[LOWER:.*]]: i32, %[[UPPER:.*]]: i32, %[[ITER:.*]]: i64) { -// CHECK: omp.simdloop for (%[[ARG_0:.*]]) : i32 = -// CHECK-SAME: (%[[LOWER]]) to (%[[UPPER]]) inclusive step (%[[LOWER]]) { +// CHECK: llvm.func @loop_nest_block_arg(%[[LOWER:.*]]: i32, %[[UPPER:.*]]: i32, %[[ITER:.*]]: i64) { +// CHECK: omp.simd { +// CHECK-NEXT: omp.loop_nest (%[[ARG_0:.*]]) : i32 = (%[[LOWER]]) +// CHECK-SAME: to (%[[UPPER]]) inclusive step (%[[LOWER]]) { // CHECK: llvm.br ^[[BB1:.*]](%[[ITER]] : i64) // CHECK: ^[[BB1]](%[[VAL_0:.*]]: i64): // CHECK: %[[VAL_1:.*]] = llvm.icmp "slt" %[[VAL_0]], %[[ITER]] : i64 @@ -157,17 +158,19 @@ func.func @threadprivate(%a: !llvm.ptr) -> () { // CHECK: llvm.br ^[[BB1]](%[[VAL_2]] : i64) // CHECK: ^[[BB3]]: // CHECK: omp.yield -func.func @simdloop_block_arg(%val : i32, %ub : i32, %i : index) { - omp.simdloop for (%arg0) : i32 = (%val) to (%ub) inclusive step (%val) { - cf.br ^bb1(%i : index) - ^bb1(%0: index): - %1 = arith.cmpi slt, %0, %i : index - cf.cond_br %1, ^bb2, ^bb3 - ^bb2: - %2 = arith.addi %0, %i : index - cf.br ^bb1(%2 : index) - ^bb3: - omp.yield +func.func @loop_nest_block_arg(%val : i32, %ub : i32, %i : index) { + omp.simd { + omp.loop_nest (%arg0) : i32 = (%val) to (%ub) inclusive step (%val) { + cf.br ^bb1(%i : index) + ^bb1(%0: index): + %1 = arith.cmpi slt, %0, %i : index + cf.cond_br %1, ^bb2, ^bb3 + ^bb2: + %2 = arith.addi %0, %i : index + cf.br ^bb1(%2 : index) + ^bb3: + omp.yield + } } return } diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir index 7f86a7f5b318..9323beadf454 100644 --- a/mlir/test/Dialect/OpenMP/invalid.mlir +++ b/mlir/test/Dialect/OpenMP/invalid.mlir @@ -243,145 +243,168 @@ llvm.func @test_omp_wsloop_dynamic_wrong_modifier3(%lb : i64, %ub : i64, %step : // ----- -func.func @omp_simdloop(%lb : index, %ub : index, %step : i32) -> () { - // expected-error @below {{op failed to verify that all of {lowerBound, upperBound, step} have same type}} - "omp.simdloop" (%lb, %ub, %step) ({ - ^bb0(%iv: index): - omp.yield - }) {operandSegmentSizes = array} : - (index, index, i32) -> () +func.func @omp_simd() -> () { + // expected-error @below {{op must be a loop wrapper}} + omp.simd { + omp.terminator + } + return +} +// ----- + +func.func @omp_simd_nested_wrapper() -> () { + // expected-error @below {{op must wrap an 'omp.loop_nest' directly}} + omp.simd { + omp.distribute { + omp.terminator + } + } return } // ----- -func.func @omp_simdloop_pretty_aligned(%lb : index, %ub : index, %step : index, - %data_var : memref) -> () { +func.func @omp_simd_pretty_aligned(%lb : index, %ub : index, %step : index, + %data_var : memref) -> () { // expected-error @below {{expected '->'}} - omp.simdloop aligned(%data_var : memref) - for (%iv) : index = (%lb) to (%ub) step (%step) { - omp.yield + omp.simd aligned(%data_var : memref) { + omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) { + omp.yield + } } return } // ----- -func.func @omp_simdloop_aligned_mismatch(%arg0 : index, %arg1 : index, - %arg2 : index, %arg3 : memref, - %arg4 : memref) -> () { +func.func @omp_simd_aligned_mismatch(%arg0 : index, %arg1 : index, + %arg2 : index, %arg3 : memref, + %arg4 : memref) -> () { // expected-error @below {{op expected as many alignment values as aligned variables}} - "omp.simdloop"(%arg0, %arg1, %arg2, %arg3, %arg4) ({ - ^bb0(%arg5: index): - "omp.yield"() : () -> () + "omp.simd"(%arg3, %arg4) ({ + omp.loop_nest (%iv) : index = (%arg0) to (%arg1) step (%arg2) { + omp.yield + } }) {alignment_values = [128], - operandSegmentSizes = array} : (index, index, index, memref, memref) -> () + operandSegmentSizes = array} : (memref, memref) -> () return } // ----- -func.func @omp_simdloop_aligned_negative(%arg0 : index, %arg1 : index, - %arg2 : index, %arg3 : memref, - %arg4 : memref) -> () { +func.func @omp_simd_aligned_negative(%arg0 : index, %arg1 : index, + %arg2 : index, %arg3 : memref, + %arg4 : memref) -> () { // expected-error @below {{op alignment should be greater than 0}} - "omp.simdloop"(%arg0, %arg1, %arg2, %arg3, %arg4) ({ - ^bb0(%arg5: index): - "omp.yield"() : () -> () - }) {alignment_values = [-1, 128], operandSegmentSizes = array} : (index, index, index, memref, memref) -> () + "omp.simd"(%arg3, %arg4) ({ + omp.loop_nest (%iv) : index = (%arg0) to (%arg1) step (%arg2) { + omp.yield + } + }) {alignment_values = [-1, 128], operandSegmentSizes = array} : (memref, memref) -> () return } // ----- -func.func @omp_simdloop_unexpected_alignment(%arg0 : index, %arg1 : index, - %arg2 : index, %arg3 : memref, - %arg4 : memref) -> () { +func.func @omp_simd_unexpected_alignment(%arg0 : index, %arg1 : index, + %arg2 : index, %arg3 : memref, + %arg4 : memref) -> () { // expected-error @below {{unexpected alignment values attribute}} - "omp.simdloop"(%arg0, %arg1, %arg2) ({ - ^bb0(%arg5: index): - "omp.yield"() : () -> () - }) {alignment_values = [1, 128], operandSegmentSizes = array} : (index, index, index) -> () + "omp.simd"() ({ + omp.loop_nest (%iv) : index = (%arg0) to (%arg1) step (%arg2) { + omp.yield + } + }) {alignment_values = [1, 128]} : () -> () return } // ----- -func.func @omp_simdloop_aligned_float(%arg0 : index, %arg1 : index, - %arg2 : index, %arg3 : memref, - %arg4 : memref) -> () { +func.func @omp_simd_aligned_float(%arg0 : index, %arg1 : index, + %arg2 : index, %arg3 : memref, + %arg4 : memref) -> () { // expected-error @below {{failed to satisfy constraint: 64-bit integer array attribute}} - "omp.simdloop"(%arg0, %arg1, %arg2, %arg3, %arg4) ({ - ^bb0(%arg5: index): - "omp.yield"() : () -> () - }) {alignment_values = [1.5, 128], operandSegmentSizes = array} : (index, index, index, memref, memref) -> () + "omp.simd"(%arg3, %arg4) ({ + omp.loop_nest (%iv) : index = (%arg0) to (%arg1) step (%arg2) { + omp.yield + } + }) {alignment_values = [1.5, 128], operandSegmentSizes = array} : (memref, memref) -> () return } // ----- -func.func @omp_simdloop_aligned_the_same_var(%arg0 : index, %arg1 : index, - %arg2 : index, %arg3 : memref, - %arg4 : memref) -> () { +func.func @omp_simd_aligned_the_same_var(%arg0 : index, %arg1 : index, + %arg2 : index, %arg3 : memref, + %arg4 : memref) -> () { // expected-error @below {{aligned variable used more than once}} - "omp.simdloop"(%arg0, %arg1, %arg2, %arg3, %arg3) ({ - ^bb0(%arg5: index): - "omp.yield"() : () -> () - }) {alignment_values = [1, 128], operandSegmentSizes = array} : (index, index, index, memref, memref) -> () + "omp.simd"(%arg3, %arg3) ({ + omp.loop_nest (%iv) : index = (%arg0) to (%arg1) step (%arg2) { + omp.yield + } + }) {alignment_values = [1, 128], operandSegmentSizes = array} : (memref, memref) -> () return } // ----- -func.func @omp_simdloop_nontemporal_the_same_var(%arg0 : index, - %arg1 : index, - %arg2 : index, - %arg3 : memref) -> () { +func.func @omp_simd_nontemporal_the_same_var(%arg0 : index, %arg1 : index, + %arg2 : index, + %arg3 : memref) -> () { // expected-error @below {{nontemporal variable used more than once}} - "omp.simdloop"(%arg0, %arg1, %arg2, %arg3, %arg3) ({ - ^bb0(%arg5: index): - "omp.yield"() : () -> () - }) {operandSegmentSizes = array} : (index, index, index, memref, memref) -> () + "omp.simd"(%arg3, %arg3) ({ + omp.loop_nest (%iv) : index = (%arg0) to (%arg1) step (%arg2) { + omp.yield + } + }) {operandSegmentSizes = array} : (memref, memref) -> () return } // ----- -func.func @omp_simdloop_order_value(%lb : index, %ub : index, %step : index) { +func.func @omp_simd_order_value(%lb : index, %ub : index, %step : index) { // expected-error @below {{invalid clause value: 'default'}} - omp.simdloop order(default) for (%iv): index = (%lb) to (%ub) step (%step) { - omp.yield + omp.simd order(default) { + omp.loop_nest (%iv) : index = (%arg0) to (%arg1) step (%arg2) { + omp.yield + } } return } // ----- -func.func @omp_simdloop_pretty_simdlen(%lb : index, %ub : index, %step : index) -> () { +func.func @omp_simd_pretty_simdlen(%lb : index, %ub : index, %step : index) -> () { // expected-error @below {{op attribute 'simdlen' failed to satisfy constraint: 64-bit signless integer attribute whose value is positive}} - omp.simdloop simdlen(0) for (%iv): index = (%lb) to (%ub) step (%step) { - omp.yield + omp.simd simdlen(0) { + omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) { + omp.yield + } } return } // ----- -func.func @omp_simdloop_pretty_safelen(%lb : index, %ub : index, %step : index) -> () { +func.func @omp_simd_pretty_safelen(%lb : index, %ub : index, %step : index) -> () { // expected-error @below {{op attribute 'safelen' failed to satisfy constraint: 64-bit signless integer attribute whose value is positive}} - omp.simdloop safelen(0) for (%iv): index = (%lb) to (%ub) step (%step) { - omp.yield + omp.simd safelen(0) { + omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) { + omp.yield + } } return } // ----- -func.func @omp_simdloop_pretty_simdlen_safelen(%lb : index, %ub : index, %step : index) -> () { - // expected-error @below {{'omp.simdloop' op simdlen clause and safelen clause are both present, but the simdlen value is not less than or equal to safelen value}} - omp.simdloop simdlen(2) safelen(1) for (%iv): index = (%lb) to (%ub) step (%step) { - omp.yield +func.func @omp_simd_pretty_simdlen_safelen(%lb : index, %ub : index, %step : index) -> () { + // expected-error @below {{op simdlen clause and safelen clause are both present, but the simdlen value is not less than or equal to safelen value}} + omp.simd simdlen(2) safelen(1) { + omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) { + omp.yield + } } return } @@ -1720,7 +1743,7 @@ func.func @taskloop(%lb: i32, %ub: i32, %step: i32) { // ----- func.func @taskloop(%lb: i32, %ub: i32, %step: i32) { - // expected-error @below {{only supported nested wrapper is 'omp.simdloop'}} + // expected-error @below {{only supported nested wrapper is 'omp.simd'}} omp.taskloop { omp.distribute { omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) { @@ -1927,7 +1950,7 @@ func.func @omp_distribute_wrapper() -> () { // ----- func.func @omp_distribute_nested_wrapper(%data_var : memref) -> () { - // expected-error @below {{only supported nested wrappers are 'omp.parallel' and 'omp.simdloop'}} + // expected-error @below {{only supported nested wrappers are 'omp.parallel' and 'omp.simd'}} "omp.distribute"() ({ "omp.wsloop"() ({ %0 = arith.constant 0 : i32 diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir index 802e1795b3ff..e2ca12afc14b 100644 --- a/mlir/test/Dialect/OpenMP/ops.mlir +++ b/mlir/test/Dialect/OpenMP/ops.mlir @@ -439,154 +439,161 @@ func.func @omp_wsloop_pretty_multiple(%lb1 : i32, %ub1 : i32, %step1 : i32, %lb2 return } -// CHECK-LABEL: omp_simdloop -func.func @omp_simdloop(%lb : index, %ub : index, %step : index) -> () { - // CHECK: omp.simdloop for (%{{.*}}) : index = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) - "omp.simdloop" (%lb, %ub, %step) ({ - ^bb0(%iv: index): - omp.yield - }) {operandSegmentSizes = array} : - (index, index, index) -> () +// CHECK-LABEL: omp_simd +func.func @omp_simd(%lb : index, %ub : index, %step : index) -> () { + // CHECK: omp.simd + "omp.simd" () ({ + "omp.loop_nest" (%lb, %ub, %step) ({ + ^bb1(%iv2: index): + "omp.yield"() : () -> () + }) : (index, index, index) -> () + "omp.terminator"() : () -> () + }) : () -> () return } -// CHECK-LABEL: omp_simdloop_aligned_list -func.func @omp_simdloop_aligned_list(%arg0 : index, %arg1 : index, %arg2 : index, - %arg3 : memref, %arg4 : memref) -> () { - // CHECK: omp.simdloop aligned(%{{.*}} : memref -> 32 : i64, +// CHECK-LABEL: omp_simd_aligned_list +func.func @omp_simd_aligned_list(%arg0 : index, %arg1 : index, %arg2 : index, + %arg3 : memref, %arg4 : memref) -> () { + // CHECK: omp.simd aligned( + // CHECK-SAME: %{{.*}} : memref -> 32 : i64, // CHECK-SAME: %{{.*}} : memref -> 128 : i64) - // CHECK-SAME: for (%{{.*}}) : index = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) { - "omp.simdloop"(%arg0, %arg1, %arg2, %arg3, %arg4) ({ - ^bb0(%arg5: index): + "omp.simd"(%arg3, %arg4) ({ + "omp.loop_nest" (%arg0, %arg1, %arg2) ({ + ^bb1(%iv2: index): "omp.yield"() : () -> () + }) : (index, index, index) -> () + "omp.terminator"() : () -> () }) {alignment_values = [32, 128], - operandSegmentSizes = array} : (index, index, index, memref, memref) -> () + operandSegmentSizes = array} : (memref, memref) -> () return } -// CHECK-LABEL: omp_simdloop_aligned_single -func.func @omp_simdloop_aligned_single(%arg0 : index, %arg1 : index, %arg2 : index, - %arg3 : memref, %arg4 : memref) -> () { - // CHECK: omp.simdloop aligned(%{{.*}} : memref -> 32 : i64) - // CHECK-SAME: for (%{{.*}}) : index = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) { - "omp.simdloop"(%arg0, %arg1, %arg2, %arg3) ({ - ^bb0(%arg5: index): +// CHECK-LABEL: omp_simd_aligned_single +func.func @omp_simd_aligned_single(%arg0 : index, %arg1 : index, %arg2 : index, + %arg3 : memref, %arg4 : memref) -> () { + // CHECK: omp.simd aligned(%{{.*}} : memref -> 32 : i64) + "omp.simd"(%arg3) ({ + "omp.loop_nest" (%arg0, %arg1, %arg2) ({ + ^bb1(%iv2: index): "omp.yield"() : () -> () + }) : (index, index, index) -> () + "omp.terminator"() : () -> () }) {alignment_values = [32], - operandSegmentSizes = array} : (index, index, index, memref) -> () + operandSegmentSizes = array} : (memref) -> () return } -// CHECK-LABEL: omp_simdloop_nontemporal_list -func.func @omp_simdloop_nontemporal_list(%arg0 : index, - %arg1 : index, - %arg2 : index, - %arg3 : memref, - %arg4 : memref) -> () { - // CHECK: omp.simdloop nontemporal(%{{.*}}, %{{.*}} : memref, memref) - // CHECK-SAME: for (%{{.*}}) : index = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) { - "omp.simdloop"(%arg0, %arg1, %arg2, %arg3, %arg4) ({ - ^bb0(%arg5: index): +// CHECK-LABEL: omp_simd_nontemporal_list +func.func @omp_simd_nontemporal_list(%arg0 : index, %arg1 : index, + %arg2 : index, %arg3 : memref, + %arg4 : memref) -> () { + // CHECK: omp.simd nontemporal(%{{.*}}, %{{.*}} : memref, memref) + "omp.simd"(%arg3, %arg4) ({ + "omp.loop_nest" (%arg0, %arg1, %arg2) ({ + ^bb1(%iv2: index): "omp.yield"() : () -> () - }) {operandSegmentSizes = array} : (index, index, index, memref, memref) -> () + }) : (index, index, index) -> () + "omp.terminator"() : () -> () + }) {operandSegmentSizes = array} : (memref, memref) -> () return } -// CHECK-LABEL: omp_simdloop_nontemporal_single -func.func @omp_simdloop_nontemporal_single(%arg0 : index, - %arg1 : index, - %arg2 : index, - %arg3 : memref, - %arg4 : memref) -> () { - // CHECK: omp.simdloop nontemporal(%{{.*}} : memref) - // CHECK-SAME: for (%{{.*}}) : index = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) { - "omp.simdloop"(%arg0, %arg1, %arg2, %arg3) ({ - ^bb0(%arg5: index): +// CHECK-LABEL: omp_simd_nontemporal_single +func.func @omp_simd_nontemporal_single(%arg0 : index, %arg1 : index, + %arg2 : index, %arg3 : memref, + %arg4 : memref) -> () { + // CHECK: omp.simd nontemporal(%{{.*}} : memref) + "omp.simd"(%arg3) ({ + "omp.loop_nest" (%arg0, %arg1, %arg2) ({ + ^bb1(%iv2: index): "omp.yield"() : () -> () - }) {operandSegmentSizes = array} : (index, index, index, memref) -> () + }) : (index, index, index) -> () + "omp.terminator"() : () -> () + }) {operandSegmentSizes = array} : (memref) -> () return } -// CHECK-LABEL: omp_simdloop_pretty -func.func @omp_simdloop_pretty(%lb : index, %ub : index, %step : index) -> () { - // CHECK: omp.simdloop for (%{{.*}}) : index = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) - omp.simdloop for (%iv) : index = (%lb) to (%ub) step (%step) { - omp.yield +// CHECK-LABEL: omp_simd_pretty +func.func @omp_simd_pretty(%lb : index, %ub : index, %step : index) -> () { + // CHECK: omp.simd { + omp.simd { + omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) { + omp.yield + } } return } -// CHECK-LABEL: func.func @omp_simdloop_pretty_aligned( -func.func @omp_simdloop_pretty_aligned(%lb : index, %ub : index, %step : index, - %data_var : memref, - %data_var1 : memref) -> () { - // CHECK: omp.simdloop aligned(%{{.*}} : memref -> 32 : i64, +// CHECK-LABEL: func.func @omp_simd_pretty_aligned( +func.func @omp_simd_pretty_aligned(%lb : index, %ub : index, %step : index, + %data_var : memref, + %data_var1 : memref) -> () { + // CHECK: omp.simd aligned( + // CHECK-SAME: %{{.*}} : memref -> 32 : i64, // CHECK-SAME: %{{.*}} : memref -> 128 : i64) - // CHECK-SAME: for (%{{.*}}) : index = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) { - omp.simdloop aligned(%data_var : memref -> 32, %data_var1 : memref -> 128) - for (%iv) : index = (%lb) to (%ub) step (%step) { + omp.simd aligned(%data_var : memref -> 32, %data_var1 : memref -> 128) { + omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) { omp.yield + } } return } -// CHECK-LABEL: omp_simdloop_pretty_if -func.func @omp_simdloop_pretty_if(%lb : index, %ub : index, %step : index, %if_cond : i1) -> () { - // CHECK: omp.simdloop if(%{{.*}}) for (%{{.*}}) : index = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) - omp.simdloop if(%if_cond) for (%iv): index = (%lb) to (%ub) step (%step) { - omp.yield +// CHECK-LABEL: omp_simd_pretty_if +func.func @omp_simd_pretty_if(%lb : index, %ub : index, %step : index, %if_cond : i1) -> () { + // CHECK: omp.simd if(%{{.*}}) + omp.simd if(%if_cond) { + omp.loop_nest (%iv): index = (%lb) to (%ub) step (%step) { + omp.yield + } } return } -// CHECK-LABEL: func.func @omp_simdloop_pretty_nontemporal -func.func @omp_simdloop_pretty_nontemporal(%lb : index, - %ub : index, - %step : index, - %data_var : memref, - %data_var1 : memref) -> () { - // CHECK: omp.simdloop nontemporal(%{{.*}}, %{{.*}} : memref, memref) - // CHECK-SAME: for (%{{.*}}) : index = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) { - omp.simdloop nontemporal(%data_var, %data_var1 : memref, memref) - for (%iv) : index = (%lb) to (%ub) step (%step) { +// CHECK-LABEL: func.func @omp_simd_pretty_nontemporal +func.func @omp_simd_pretty_nontemporal(%lb : index, %ub : index, %step : index, + %data_var : memref, + %data_var1 : memref) -> () { + // CHECK: omp.simd nontemporal(%{{.*}}, %{{.*}} : memref, memref) + omp.simd nontemporal(%data_var, %data_var1 : memref, memref) { + omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) { omp.yield - } - return -} -// CHECK-LABEL: omp_simdloop_pretty_order -func.func @omp_simdloop_pretty_order(%lb : index, %ub : index, %step : index) -> () { - // CHECK: omp.simdloop order(concurrent) - // CHECK-SAME: for (%{{.*}}) : index = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) - omp.simdloop order(concurrent) for (%iv): index = (%lb) to (%ub) step (%step) { - omp.yield + } } return } -// CHECK-LABEL: omp_simdloop_pretty_simdlen -func.func @omp_simdloop_pretty_simdlen(%lb : index, %ub : index, %step : index) -> () { - // CHECK: omp.simdloop simdlen(2) for (%{{.*}}) : index = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) - omp.simdloop simdlen(2) for (%iv): index = (%lb) to (%ub) step (%step) { - omp.yield +// CHECK-LABEL: omp_simd_pretty_order +func.func @omp_simd_pretty_order(%lb : index, %ub : index, %step : index) -> () { + // CHECK: omp.simd order(concurrent) + omp.simd order(concurrent) { + omp.loop_nest (%iv): index = (%lb) to (%ub) step (%step) { + omp.yield + } } return } -// CHECK-LABEL: omp_simdloop_pretty_safelen -func.func @omp_simdloop_pretty_safelen(%lb : index, %ub : index, %step : index) -> () { - // CHECK: omp.simdloop safelen(2) for (%{{.*}}) : index = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) - omp.simdloop safelen(2) for (%iv): index = (%lb) to (%ub) step (%step) { - omp.yield +// CHECK-LABEL: omp_simd_pretty_simdlen +func.func @omp_simd_pretty_simdlen(%lb : index, %ub : index, %step : index) -> () { + // CHECK: omp.simd simdlen(2) + omp.simd simdlen(2) { + omp.loop_nest (%iv): index = (%lb) to (%ub) step (%step) { + omp.yield + } } return } -// CHECK-LABEL: omp_simdloop_pretty_multiple -func.func @omp_simdloop_pretty_multiple(%lb1 : index, %ub1 : index, %step1 : index, %lb2 : index, %ub2 : index, %step2 : index) -> () { - // CHECK: omp.simdloop for (%{{.*}}, %{{.*}}) : index = (%{{.*}}, %{{.*}}) to (%{{.*}}, %{{.*}}) step (%{{.*}}, %{{.*}}) - omp.simdloop for (%iv1, %iv2) : index = (%lb1, %lb2) to (%ub1, %ub2) step (%step1, %step2) { - omp.yield +// CHECK-LABEL: omp_simd_pretty_safelen +func.func @omp_simd_pretty_safelen(%lb : index, %ub : index, %step : index) -> () { + // CHECK: omp.simd safelen(2) + omp.simd safelen(2) { + omp.loop_nest (%iv): index = (%lb) to (%ub) step (%step) { + omp.yield + } } return } @@ -633,15 +640,13 @@ func.func @omp_distribute(%chunk_size : i32, %data_var : memref, %arg0 : i3 } // CHECK: omp.distribute omp.distribute { - // TODO Remove induction variables from omp.simdloop. - omp.simdloop for (%iv) : i32 = (%arg0) to (%arg0) step (%arg0) { + omp.simd { omp.loop_nest (%iv2) : i32 = (%arg0) to (%arg0) step (%arg0) { omp.yield } - omp.yield } } -return + return } @@ -2170,14 +2175,11 @@ func.func @omp_taskloop(%lb: i32, %ub: i32, %step: i32) -> () { // CHECK: omp.taskloop { omp.taskloop { - // TODO Remove induction variables from omp.simdloop. - omp.simdloop for (%iv) : i32 = (%lb) to (%ub) step (%step) { + omp.simd { omp.loop_nest (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) { // CHECK: omp.yield omp.yield } - // CHECK: omp.yield - omp.yield } } diff --git a/mlir/test/Target/LLVMIR/openmp-llvm.mlir b/mlir/test/Target/LLVMIR/openmp-llvm.mlir index 4cb99c1f1a28..d1390022c1dc 100644 --- a/mlir/test/Target/LLVMIR/openmp-llvm.mlir +++ b/mlir/test/Target/LLVMIR/openmp-llvm.mlir @@ -638,10 +638,10 @@ llvm.func @test_omp_wsloop_guided_simd(%lb : i64, %ub : i64, %step : i64) -> () // ----- -// CHECK-LABEL: @simdloop_simple -llvm.func @simdloop_simple(%lb : i64, %ub : i64, %step : i64, %arg0: !llvm.ptr) { - "omp.simdloop" (%lb, %ub, %step) ({ - ^bb0(%iv: i64): +// CHECK-LABEL: @simd_simple +llvm.func @simd_simple(%lb : i64, %ub : i64, %step : i64, %arg0: !llvm.ptr) { + "omp.simd" () ({ + omp.loop_nest (%iv) : i64 = (%lb) to (%ub) step (%step) { %3 = llvm.mlir.constant(2.000000e+00 : f32) : f32 // The form of the emitted IR is controlled by OpenMPIRBuilder and // tested there. Just check that the right metadata is added. @@ -649,8 +649,9 @@ llvm.func @simdloop_simple(%lb : i64, %ub : i64, %step : i64, %arg0: !llvm.ptr) %4 = llvm.getelementptr %arg0[%iv] : (!llvm.ptr, i64) -> !llvm.ptr, f32 llvm.store %3, %4 : f32, !llvm.ptr omp.yield - }) {operandSegmentSizes = array} : - (i64, i64, i64) -> () + } + "omp.terminator"() : () -> () + }) : () -> () llvm.return } @@ -659,34 +660,36 @@ llvm.func @simdloop_simple(%lb : i64, %ub : i64, %step : i64, %arg0: !llvm.ptr) // ----- -// CHECK-LABEL: @simdloop_simple_multiple -llvm.func @simdloop_simple_multiple(%lb1 : i64, %ub1 : i64, %step1 : i64, %lb2 : i64, %ub2 : i64, %step2 : i64, %arg0: !llvm.ptr, %arg1: !llvm.ptr) { - omp.simdloop for (%iv1, %iv2) : i64 = (%lb1, %lb2) to (%ub1, %ub2) step (%step1, %step2) { - %3 = llvm.mlir.constant(2.000000e+00 : f32) : f32 - // The form of the emitted IR is controlled by OpenMPIRBuilder and - // tested there. Just check that the right metadata is added and collapsed - // loop bound is generated (Collapse clause is represented as a loop with - // list of indices, bounds and steps where the size of the list is equal - // to the collapse value.) - // CHECK: icmp slt i64 - // CHECK-COUNT-3: select - // CHECK: %[[TRIPCOUNT0:.*]] = select - // CHECK: br label %[[PREHEADER:.*]] - // CHECK: [[PREHEADER]]: - // CHECK: icmp slt i64 - // CHECK-COUNT-3: select - // CHECK: %[[TRIPCOUNT1:.*]] = select - // CHECK: mul nuw i64 %[[TRIPCOUNT0]], %[[TRIPCOUNT1]] - // CHECK: br label %[[COLLAPSED_PREHEADER:.*]] - // CHECK: [[COLLAPSED_PREHEADER]]: - // CHECK: br label %[[COLLAPSED_HEADER:.*]] - // CHECK: llvm.access.group - // CHECK-NEXT: llvm.access.group - %4 = llvm.getelementptr %arg0[%iv1] : (!llvm.ptr, i64) -> !llvm.ptr, f32 - %5 = llvm.getelementptr %arg1[%iv2] : (!llvm.ptr, i64) -> !llvm.ptr, f32 - llvm.store %3, %4 : f32, !llvm.ptr - llvm.store %3, %5 : f32, !llvm.ptr - omp.yield +// CHECK-LABEL: @simd_simple_multiple +llvm.func @simd_simple_multiple(%lb1 : i64, %ub1 : i64, %step1 : i64, %lb2 : i64, %ub2 : i64, %step2 : i64, %arg0: !llvm.ptr, %arg1: !llvm.ptr) { + omp.simd { + omp.loop_nest (%iv1, %iv2) : i64 = (%lb1, %lb2) to (%ub1, %ub2) step (%step1, %step2) { + %3 = llvm.mlir.constant(2.000000e+00 : f32) : f32 + // The form of the emitted IR is controlled by OpenMPIRBuilder and + // tested there. Just check that the right metadata is added and collapsed + // loop bound is generated (Collapse clause is represented as a loop with + // list of indices, bounds and steps where the size of the list is equal + // to the collapse value.) + // CHECK: icmp slt i64 + // CHECK-COUNT-3: select + // CHECK: %[[TRIPCOUNT0:.*]] = select + // CHECK: br label %[[PREHEADER:.*]] + // CHECK: [[PREHEADER]]: + // CHECK: icmp slt i64 + // CHECK-COUNT-3: select + // CHECK: %[[TRIPCOUNT1:.*]] = select + // CHECK: mul nuw i64 %[[TRIPCOUNT0]], %[[TRIPCOUNT1]] + // CHECK: br label %[[COLLAPSED_PREHEADER:.*]] + // CHECK: [[COLLAPSED_PREHEADER]]: + // CHECK: br label %[[COLLAPSED_HEADER:.*]] + // CHECK: llvm.access.group + // CHECK-NEXT: llvm.access.group + %4 = llvm.getelementptr %arg0[%iv1] : (!llvm.ptr, i64) -> !llvm.ptr, f32 + %5 = llvm.getelementptr %arg1[%iv2] : (!llvm.ptr, i64) -> !llvm.ptr, f32 + llvm.store %3, %4 : f32, !llvm.ptr + llvm.store %3, %5 : f32, !llvm.ptr + omp.yield + } } llvm.return } @@ -695,19 +698,21 @@ llvm.func @simdloop_simple_multiple(%lb1 : i64, %ub1 : i64, %step1 : i64, %lb2 : // ----- -// CHECK-LABEL: @simdloop_simple_multiple_simdlen -llvm.func @simdloop_simple_multiple_simdlen(%lb1 : i64, %ub1 : i64, %step1 : i64, %lb2 : i64, %ub2 : i64, %step2 : i64, %arg0: !llvm.ptr, %arg1: !llvm.ptr) { - omp.simdloop simdlen(2) for (%iv1, %iv2) : i64 = (%lb1, %lb2) to (%ub1, %ub2) step (%step1, %step2) { - %3 = llvm.mlir.constant(2.000000e+00 : f32) : f32 - // The form of the emitted IR is controlled by OpenMPIRBuilder and - // tested there. Just check that the right metadata is added. - // CHECK: llvm.access.group - // CHECK-NEXT: llvm.access.group - %4 = llvm.getelementptr %arg0[%iv1] : (!llvm.ptr, i64) -> !llvm.ptr, f32 - %5 = llvm.getelementptr %arg1[%iv2] : (!llvm.ptr, i64) -> !llvm.ptr, f32 - llvm.store %3, %4 : f32, !llvm.ptr - llvm.store %3, %5 : f32, !llvm.ptr - omp.yield +// CHECK-LABEL: @simd_simple_multiple_simdlen +llvm.func @simd_simple_multiple_simdlen(%lb1 : i64, %ub1 : i64, %step1 : i64, %lb2 : i64, %ub2 : i64, %step2 : i64, %arg0: !llvm.ptr, %arg1: !llvm.ptr) { + omp.simd simdlen(2) { + omp.loop_nest (%iv1, %iv2) : i64 = (%lb1, %lb2) to (%ub1, %ub2) step (%step1, %step2) { + %3 = llvm.mlir.constant(2.000000e+00 : f32) : f32 + // The form of the emitted IR is controlled by OpenMPIRBuilder and + // tested there. Just check that the right metadata is added. + // CHECK: llvm.access.group + // CHECK-NEXT: llvm.access.group + %4 = llvm.getelementptr %arg0[%iv1] : (!llvm.ptr, i64) -> !llvm.ptr, f32 + %5 = llvm.getelementptr %arg1[%iv2] : (!llvm.ptr, i64) -> !llvm.ptr, f32 + llvm.store %3, %4 : f32, !llvm.ptr + llvm.store %3, %5 : f32, !llvm.ptr + omp.yield + } } llvm.return } @@ -717,15 +722,17 @@ llvm.func @simdloop_simple_multiple_simdlen(%lb1 : i64, %ub1 : i64, %step1 : i64 // ----- -// CHECK-LABEL: @simdloop_simple_multiple_safelen -llvm.func @simdloop_simple_multiple_safelen(%lb1 : i64, %ub1 : i64, %step1 : i64, %lb2 : i64, %ub2 : i64, %step2 : i64, %arg0: !llvm.ptr, %arg1: !llvm.ptr) { - omp.simdloop safelen(2) for (%iv1, %iv2) : i64 = (%lb1, %lb2) to (%ub1, %ub2) step (%step1, %step2) { - %3 = llvm.mlir.constant(2.000000e+00 : f32) : f32 - %4 = llvm.getelementptr %arg0[%iv1] : (!llvm.ptr, i64) -> !llvm.ptr, f32 - %5 = llvm.getelementptr %arg1[%iv2] : (!llvm.ptr, i64) -> !llvm.ptr, f32 - llvm.store %3, %4 : f32, !llvm.ptr - llvm.store %3, %5 : f32, !llvm.ptr - omp.yield +// CHECK-LABEL: @simd_simple_multiple_safelen +llvm.func @simd_simple_multiple_safelen(%lb1 : i64, %ub1 : i64, %step1 : i64, %lb2 : i64, %ub2 : i64, %step2 : i64, %arg0: !llvm.ptr, %arg1: !llvm.ptr) { + omp.simd safelen(2) { + omp.loop_nest (%iv1, %iv2) : i64 = (%lb1, %lb2) to (%ub1, %ub2) step (%step1, %step2) { + %3 = llvm.mlir.constant(2.000000e+00 : f32) : f32 + %4 = llvm.getelementptr %arg0[%iv1] : (!llvm.ptr, i64) -> !llvm.ptr, f32 + %5 = llvm.getelementptr %arg1[%iv2] : (!llvm.ptr, i64) -> !llvm.ptr, f32 + llvm.store %3, %4 : f32, !llvm.ptr + llvm.store %3, %5 : f32, !llvm.ptr + omp.yield + } } llvm.return } @@ -734,15 +741,17 @@ llvm.func @simdloop_simple_multiple_safelen(%lb1 : i64, %ub1 : i64, %step1 : i64 // ----- -// CHECK-LABEL: @simdloop_simple_multiple_simdlen_safelen -llvm.func @simdloop_simple_multiple_simdlen_safelen(%lb1 : i64, %ub1 : i64, %step1 : i64, %lb2 : i64, %ub2 : i64, %step2 : i64, %arg0: !llvm.ptr, %arg1: !llvm.ptr) { - omp.simdloop simdlen(1) safelen(2) for (%iv1, %iv2) : i64 = (%lb1, %lb2) to (%ub1, %ub2) step (%step1, %step2) { - %3 = llvm.mlir.constant(2.000000e+00 : f32) : f32 - %4 = llvm.getelementptr %arg0[%iv1] : (!llvm.ptr, i64) -> !llvm.ptr, f32 - %5 = llvm.getelementptr %arg1[%iv2] : (!llvm.ptr, i64) -> !llvm.ptr, f32 - llvm.store %3, %4 : f32, !llvm.ptr - llvm.store %3, %5 : f32, !llvm.ptr - omp.yield +// CHECK-LABEL: @simd_simple_multiple_simdlen_safelen +llvm.func @simd_simple_multiple_simdlen_safelen(%lb1 : i64, %ub1 : i64, %step1 : i64, %lb2 : i64, %ub2 : i64, %step2 : i64, %arg0: !llvm.ptr, %arg1: !llvm.ptr) { + omp.simd simdlen(1) safelen(2) { + omp.loop_nest (%iv1, %iv2) : i64 = (%lb1, %lb2) to (%ub1, %ub2) step (%step1, %step2) { + %3 = llvm.mlir.constant(2.000000e+00 : f32) : f32 + %4 = llvm.getelementptr %arg0[%iv1] : (!llvm.ptr, i64) -> !llvm.ptr, f32 + %5 = llvm.getelementptr %arg1[%iv2] : (!llvm.ptr, i64) -> !llvm.ptr, f32 + llvm.store %3, %4 : f32, !llvm.ptr + llvm.store %3, %5 : f32, !llvm.ptr + omp.yield + } } llvm.return } @@ -751,8 +760,8 @@ llvm.func @simdloop_simple_multiple_simdlen_safelen(%lb1 : i64, %ub1 : i64, %ste // ----- -// CHECK-LABEL: @simdloop_if -llvm.func @simdloop_if(%arg0: !llvm.ptr {fir.bindc_name = "n"}, %arg1: !llvm.ptr {fir.bindc_name = "threshold"}) { +// CHECK-LABEL: @simd_if +llvm.func @simd_if(%arg0: !llvm.ptr {fir.bindc_name = "n"}, %arg1: !llvm.ptr {fir.bindc_name = "threshold"}) { %0 = llvm.mlir.constant(1 : i64) : i64 %1 = llvm.alloca %0 x i32 {adapt.valuebyref, in_type = i32, operandSegmentSizes = array} : (i64) -> !llvm.ptr %2 = llvm.mlir.constant(1 : i64) : i64 @@ -763,12 +772,14 @@ llvm.func @simdloop_if(%arg0: !llvm.ptr {fir.bindc_name = "n"}, %arg1: !llvm.ptr %7 = llvm.load %arg0 : !llvm.ptr -> i32 %8 = llvm.load %arg1 : !llvm.ptr -> i32 %9 = llvm.icmp "sge" %7, %8 : i32 - omp.simdloop if(%9) for (%arg2) : i32 = (%4) to (%5) inclusive step (%6) { - // The form of the emitted IR is controlled by OpenMPIRBuilder and - // tested there. Just check that the right metadata is added. - // CHECK: llvm.access.group - llvm.store %arg2, %1 : i32, !llvm.ptr - omp.yield + omp.simd if(%9) { + omp.loop_nest (%arg2) : i32 = (%4) to (%5) inclusive step (%6) { + // The form of the emitted IR is controlled by OpenMPIRBuilder and + // tested there. Just check that the right metadata is added. + // CHECK: llvm.access.group + llvm.store %arg2, %1 : i32, !llvm.ptr + omp.yield + } } llvm.return } -- GitLab From 16b0be613205a37519e2bfec1a904ceaa89636e7 Mon Sep 17 00:00:00 2001 From: Sergio Afonso Date: Wed, 17 Apr 2024 11:30:11 +0100 Subject: [PATCH 031/862] [MLIR][OpenMP] NFC: Remove LoopControl parsing/printing code (#88909) This patch removes the LoopControl parsing/printing functions that are no longer used after transitioning `omp.simdloop` and `omp.taskloop` into loop wrapper operations. --- mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp | 52 -------------------- 1 file changed, 52 deletions(-) diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp index caf0ac3f8601..5d2281ce6094 100644 --- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp +++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp @@ -1548,58 +1548,6 @@ void printWsloop(OpAsmPrinter &p, Operation *op, Region ®ion, p.printRegion(region, /*printEntryBlockArgs=*/false); } -/// loop-control ::= `(` ssa-id-list `)` `:` type `=` loop-bounds -/// loop-bounds := `(` ssa-id-list `)` to `(` ssa-id-list `)` inclusive? steps -/// steps := `step` `(`ssa-id-list`)` -ParseResult -parseLoopControl(OpAsmParser &parser, Region ®ion, - SmallVectorImpl &lowerBound, - SmallVectorImpl &upperBound, - SmallVectorImpl &steps, - SmallVectorImpl &loopVarTypes, UnitAttr &inclusive) { - // Parse an opening `(` followed by induction variables followed by `)` - SmallVector ivs; - Type loopVarType; - if (parser.parseArgumentList(ivs, OpAsmParser::Delimiter::Paren) || - parser.parseColonType(loopVarType) || - // Parse loop bounds. - parser.parseEqual() || - parser.parseOperandList(lowerBound, ivs.size(), - OpAsmParser::Delimiter::Paren) || - parser.parseKeyword("to") || - parser.parseOperandList(upperBound, ivs.size(), - OpAsmParser::Delimiter::Paren)) - return failure(); - - if (succeeded(parser.parseOptionalKeyword("inclusive"))) - inclusive = UnitAttr::get(parser.getBuilder().getContext()); - - // Parse step values. - if (parser.parseKeyword("step") || - parser.parseOperandList(steps, ivs.size(), OpAsmParser::Delimiter::Paren)) - return failure(); - - // Now parse the body. - loopVarTypes = SmallVector(ivs.size(), loopVarType); - for (auto &iv : ivs) - iv.type = loopVarType; - - return parser.parseRegion(region, ivs); -} - -void printLoopControl(OpAsmPrinter &p, Operation *op, Region ®ion, - ValueRange lowerBound, ValueRange upperBound, - ValueRange steps, TypeRange loopVarTypes, - UnitAttr inclusive) { - auto args = region.front().getArguments(); - p << " (" << args << ") : " << args[0].getType() << " = (" << lowerBound - << ") to (" << upperBound << ") "; - if (inclusive) - p << "inclusive "; - p << "step (" << steps << ") "; - p.printRegion(region, /*printEntryBlockArgs=*/false); -} - //===----------------------------------------------------------------------===// // Simd construct [2.9.3.1] //===----------------------------------------------------------------------===// -- GitLab From a02019960b1a693320cd43b0ed6653d95877b94f Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Wed, 17 Apr 2024 18:15:26 +0800 Subject: [PATCH 032/862] [RISCV] Assert only valid AVLs in doLocalPostpass are X0 or virtual regs. NFC In vxrm.mir we were running RISCVInsertVSETVLI on pseudos that already had vsetvlis inserted and their AVLs set to $noreg. (This happened to work since doLocalPostpass got rid of the extra vsetvli) This removes the vsetvlis from the test and enforces that the only valid AVLs we work with are either X0 or virtual registers (or $noreg before emitVSETVLIs), since we don't handle physical registers properly in doLocalPostpass. --- llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp | 16 +++++++--------- llvm/test/CodeGen/RISCV/rvv/vxrm.mir | 8 ++++---- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp index a54a1148cf28..6e45f0c703ce 100644 --- a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp +++ b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp @@ -468,6 +468,7 @@ public: bool isUnknown() const { return State == Unknown; } void setAVLReg(Register Reg) { + assert(Reg.isVirtual() || Reg == RISCV::X0 || Reg == RISCV::NoRegister); AVLReg = Reg; State = AVLIsReg; } @@ -1514,12 +1515,9 @@ static bool canMutatePriorConfig(const MachineInstr &PrevMI, // If the AVL is a register, we need to make sure MI's AVL dominates PrevMI. // For now just check that PrevMI uses the same virtual register. - if (AVL.isReg() && AVL.getReg() != RISCV::X0) { - if (AVL.getReg().isPhysical()) - return false; - if (!PrevAVL.isReg() || PrevAVL.getReg() != AVL.getReg()) - return false; - } + if (AVL.isReg() && AVL.getReg() != RISCV::X0 && + (!PrevAVL.isReg() || PrevAVL.getReg() != AVL.getReg())) + return false; } assert(PrevMI.getOperand(2).isImm() && MI.getOperand(2).isImm()); @@ -1543,9 +1541,9 @@ void RISCVInsertVSETVLI::doLocalPostpass(MachineBasicBlock &MBB) { continue; } - Register VRegDef = MI.getOperand(0).getReg(); - if (VRegDef != RISCV::X0 && - !(VRegDef.isVirtual() && MRI->use_nodbg_empty(VRegDef))) + Register RegDef = MI.getOperand(0).getReg(); + assert(RegDef == RISCV::X0 || RegDef.isVirtual()); + if (RegDef != RISCV::X0 && !MRI->use_nodbg_empty(RegDef)) Used.demandVL(); if (NextMI) { diff --git a/llvm/test/CodeGen/RISCV/rvv/vxrm.mir b/llvm/test/CodeGen/RISCV/rvv/vxrm.mir index 64e191887e09..a588677bec8e 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vxrm.mir +++ b/llvm/test/CodeGen/RISCV/rvv/vxrm.mir @@ -11,9 +11,9 @@ body: | ; MIR-LABEL: name: verify_vxrm ; MIR: liveins: $v8, $v9, $x10 ; MIR-NEXT: {{ $}} - ; MIR-NEXT: dead $x0 = PseudoVSETVLI renamable $x10, 197 /* e8, mf8, ta, ma */, implicit-def $vl, implicit-def $vtype + ; MIR-NEXT: dead $x0 = PseudoVSETVLI killed renamable $x10, 197 /* e8, mf8, ta, ma */, implicit-def $vl, implicit-def $vtype ; MIR-NEXT: WriteVXRMImm 0, implicit-def $vxrm - ; MIR-NEXT: renamable $v8 = PseudoVAADD_VV_MF8 undef $v8, renamable $v8, renamable $v9, 0, $noreg, 3 /* e8 */, 0 /* tu, mu */, implicit $vl, implicit $vtype, implicit $vxrm + ; MIR-NEXT: renamable $v8 = PseudoVAADD_VV_MF8 undef $v8, killed renamable $v8, killed renamable $v9, 0, $noreg, 3 /* e8 */, 0 /* tu, mu */, implicit $vl, implicit $vtype, implicit $vxrm ; MIR-NEXT: PseudoRET implicit $v8 ; ASM-LABEL: verify_vxrm: ; ASM: # %bb.0: @@ -23,8 +23,8 @@ body: | ; ASM-NEXT: ret %0:vr = COPY $v8 %1:vr = COPY $v9 - dead $x0 = PseudoVSETVLI killed renamable $x10, 197 /* e8, mf8, ta, ma */, implicit-def $vl, implicit-def $vtype + %2:gprnox0 = COPY $x10 %pt:vr = IMPLICIT_DEF - renamable $v8 = PseudoVAADD_VV_MF8 %pt, killed renamable $v8, killed renamable $v9, 0, $noreg, 3 /* e8 */, 0 + renamable $v8 = PseudoVAADD_VV_MF8 %pt, %0, %1, 0, %2, 3 /* e8 */, 0 PseudoRET implicit $v8 ... -- GitLab From a634f3ef39c0c547b87f1ee4ebe02ee3a256587f Mon Sep 17 00:00:00 2001 From: Stephen Tozer Date: Wed, 17 Apr 2024 11:39:18 +0100 Subject: [PATCH 033/862] [RemoveDIs] Update update_test_checks script to recognize dbg_records (#87388) As we've added new IR elements for the RemoveDIs project, we need the update_test_checks script to understand them. For the records themselves this is already done automatically, but their metadata arguments are not recognized as such due to lacking the `metadata` prefix, which means they won't be checked by the script. This patch fixes this by adding a check for all `![0-9]+` patterns as long as they are not at the start of a line (which avoids matching global values). --- .../Inputs/various_ir_values_dbgrecords.ll | 168 ++++++++++ .../various_ir_values_dbgrecords.ll.expected | 238 ++++++++++++++ ...s_ir_values_dbgrecords.ll.funcsig.expected | 240 ++++++++++++++ ...ues_dbgrecords.ll.funcsig.globals.expected | 309 ++++++++++++++++++ ...s_dbgrecords.ll.funcsig.noglobals.expected | 238 ++++++++++++++ ...ords.ll.funcsig.transitiveglobals.expected | 299 +++++++++++++++++ .../various_ir_values_dbgrecords.test | 24 ++ llvm/utils/UpdateTestChecks/common.py | 2 +- 8 files changed, 1517 insertions(+), 1 deletion(-) create mode 100644 llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll create mode 100644 llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.expected create mode 100644 llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.funcsig.expected create mode 100644 llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.funcsig.globals.expected create mode 100644 llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.funcsig.noglobals.expected create mode 100644 llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.funcsig.transitiveglobals.expected create mode 100644 llvm/test/tools/UpdateTestChecks/update_test_checks/various_ir_values_dbgrecords.test diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll new file mode 100644 index 000000000000..9a9cc0a06936 --- /dev/null +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll @@ -0,0 +1,168 @@ +; Just run it through opt, no passes needed. +; RUN: opt < %s -S --write-experimental-debuginfo=true | FileCheck %s + +; ModuleID = 'various_ir_values.c' +source_filename = "various_ir_values.c" +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +; Function Attrs: nounwind uwtable +define dso_local void @foo(ptr %A) #0 !dbg !7 { +entry: + %A.addr = alloca ptr, align 8, !DIAssignID !16 + %i = alloca i32, align 4 + #dbg_assign(i1 undef, !13, !DIExpression(), !16, ptr %A.addr, !DIExpression(), !17) + store ptr %A, ptr %A.addr, align 8, !tbaa !18 + #dbg_declare(ptr %A.addr, !13, !DIExpression(), !17) + call void @llvm.lifetime.start.p0(i64 4, ptr %i) #2, !dbg !22 + #dbg_declare(ptr %i, !14, !DIExpression(), !23) + store i32 0, ptr %i, align 4, !dbg !23, !tbaa !24 + br label %for.cond, !dbg !22 + +for.cond: ; preds = %for.inc, %entry + %0 = load i32, ptr %i, align 4, !dbg !26, !tbaa !24 + %1 = load ptr, ptr %A.addr, align 8, !dbg !28, !tbaa !18 + %2 = load i32, ptr %1, align 4, !dbg !29, !tbaa !24 + %cmp = icmp slt i32 %0, %2, !dbg !30 + br i1 %cmp, label %for.body, label %for.cond.cleanup, !dbg !31, !prof !32 + +for.cond.cleanup: ; preds = %for.cond + call void @llvm.lifetime.end.p0(i64 4, ptr %i) #2, !dbg !33 + br label %for.end + +for.body: ; preds = %for.cond + %3 = load ptr, ptr %A.addr, align 8, !dbg !34, !tbaa !18 + %4 = load i32, ptr %i, align 4, !dbg !35, !tbaa !24 + %idxprom = sext i32 %4 to i64, !dbg !34 + %arrayidx = getelementptr inbounds i32, ptr %3, i64 %idxprom, !dbg !34 + store i32 0, ptr %arrayidx, align 4, !dbg !36, !tbaa !24 + br label %for.inc, !dbg !34 + +for.inc: ; preds = %for.body + %5 = load i32, ptr %i, align 4, !dbg !37, !tbaa !24 + %inc = add nsw i32 %5, 1, !dbg !37 + store i32 %inc, ptr %i, align 4, !dbg !37, !tbaa !24 + br label %for.cond, !dbg !33, !llvm.loop !38 + +for.end: ; preds = %for.cond.cleanup + ret void, !dbg !40 +} + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) +declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1 + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) +declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1 + +; Function Attrs: nounwind uwtable +define dso_local void @bar(ptr %A) #0 !dbg !41 { +entry: + %A.addr = alloca ptr, align 8 + %i = alloca i32, align 4 + store ptr %A, ptr %A.addr, align 8, !tbaa !18 + #dbg_declare(ptr %A.addr, !43, !DIExpression(), !46) + call void @llvm.lifetime.start.p0(i64 4, ptr %i) #2, !dbg !47 + #dbg_declare(ptr %i, !44, !DIExpression(), !48) + store i32 0, ptr %i, align 4, !dbg !48, !tbaa !24 + br label %for.cond, !dbg !47 + +for.cond: ; preds = %for.inc, %entry + %0 = load i32, ptr %i, align 4, !dbg !49, !tbaa !24 + %1 = load ptr, ptr %A.addr, align 8, !dbg !51, !tbaa !18 + %2 = load i32, ptr %1, align 4, !dbg !52, !tbaa !24 + %cmp = icmp slt i32 %0, %2, !dbg !53 + br i1 %cmp, label %for.body, label %for.cond.cleanup, !dbg !54 + +for.cond.cleanup: ; preds = %for.cond + call void @llvm.lifetime.end.p0(i64 4, ptr %i) #2, !dbg !55 + br label %for.end + +for.body: ; preds = %for.cond + %3 = load ptr, ptr %A.addr, align 8, !dbg !56, !tbaa !18 + %4 = load i32, ptr %i, align 4, !dbg !57, !tbaa !24 + %idxprom = sext i32 %4 to i64, !dbg !56 + %arrayidx = getelementptr inbounds i32, ptr %3, i64 %idxprom, !dbg !56 + store i32 0, ptr %arrayidx, align 4, !dbg !58, !tbaa !24 + br label %for.inc, !dbg !56 + +for.inc: ; preds = %for.body + %5 = load i32, ptr %i, align 4, !dbg !59, !tbaa !24 + %inc = add nsw i32 %5, 1, !dbg !59 + store i32 %inc, ptr %i, align 4, !dbg !59, !tbaa !24 + br label %for.cond, !dbg !55, !llvm.loop !60 + +for.end: ; preds = %for.cond.cleanup + ret void, !dbg !62 +} + +attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #1 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) } +attributes #2 = { nounwind } + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!3, !4, !5} +!llvm.ident = !{!6} + +!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 11.0.0 (git@github.com:llvm/llvm-project.git 1d5da8cd30fce1c0a2c2fa6ba656dbfaa36192c8)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, splitDebugInlining: false, nameTableKind: None) +!1 = !DIFile(filename: "various_ir_values.c", directory: "/data/build/llvm-project") +!2 = !{} +!3 = !{i32 7, !"Dwarf Version", i32 4} +!4 = !{i32 2, !"Debug Info Version", i32 3} +!5 = !{i32 1, !"wchar_size", i32 4} +!6 = !{!"clang version 11.0.0 (git@github.com:llvm/llvm-project.git 1d5da8cd30fce1c0a2c2fa6ba656dbfaa36192c8)"} +!7 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !8, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !12) +!8 = !DISubroutineType(types: !9) +!9 = !{null, !10} +!10 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !11, size: 64) +!11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!12 = !{!13, !14} +!13 = !DILocalVariable(name: "A", arg: 1, scope: !7, file: !1, line: 1, type: !10) +!14 = !DILocalVariable(name: "i", scope: !15, file: !1, line: 3, type: !11) +!15 = distinct !DILexicalBlock(scope: !7, file: !1, line: 3, column: 3) +!16 = distinct !DIAssignID() +!17 = !DILocation(line: 1, column: 15, scope: !7) +!18 = !{!19, !19, i64 0} +!19 = !{!"any pointer", !20, i64 0} +!20 = !{!"omnipotent char", !21, i64 0} +!21 = !{!"Simple C/C++ TBAA"} +!22 = !DILocation(line: 3, column: 8, scope: !15) +!23 = !DILocation(line: 3, column: 12, scope: !15) +!24 = !{!25, !25, i64 0} +!25 = !{!"int", !20, i64 0} +!26 = !DILocation(line: 3, column: 19, scope: !27) +!27 = distinct !DILexicalBlock(scope: !15, file: !1, line: 3, column: 3) +!28 = !DILocation(line: 3, column: 24, scope: !27) +!29 = !DILocation(line: 3, column: 23, scope: !27) +!30 = !DILocation(line: 3, column: 21, scope: !27) +!31 = !DILocation(line: 3, column: 3, scope: !15) +!32 = !{!"branch_weights", i32 1, i32 1048575} +!33 = !DILocation(line: 3, column: 3, scope: !27) +!34 = !DILocation(line: 4, column: 5, scope: !27) +!35 = !DILocation(line: 4, column: 7, scope: !27) +!36 = !DILocation(line: 4, column: 10, scope: !27) +!37 = !DILocation(line: 3, column: 27, scope: !27) +!38 = distinct !{!38, !31, !39} +!39 = !DILocation(line: 4, column: 12, scope: !15) +!40 = !DILocation(line: 5, column: 1, scope: !7) +!41 = distinct !DISubprogram(name: "bar", scope: !1, file: !1, line: 7, type: !8, scopeLine: 7, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !42) +!42 = !{!43, !44} +!43 = !DILocalVariable(name: "A", arg: 1, scope: !41, file: !1, line: 7, type: !10) +!44 = !DILocalVariable(name: "i", scope: !45, file: !1, line: 9, type: !11) +!45 = distinct !DILexicalBlock(scope: !41, file: !1, line: 9, column: 3) +!46 = !DILocation(line: 7, column: 15, scope: !41) +!47 = !DILocation(line: 9, column: 8, scope: !45) +!48 = !DILocation(line: 9, column: 12, scope: !45) +!49 = !DILocation(line: 9, column: 19, scope: !50) +!50 = distinct !DILexicalBlock(scope: !45, file: !1, line: 9, column: 3) +!51 = !DILocation(line: 9, column: 24, scope: !50) +!52 = !DILocation(line: 9, column: 23, scope: !50) +!53 = !DILocation(line: 9, column: 21, scope: !50) +!54 = !DILocation(line: 9, column: 3, scope: !45) +!55 = !DILocation(line: 9, column: 3, scope: !50) +!56 = !DILocation(line: 10, column: 5, scope: !50) +!57 = !DILocation(line: 10, column: 7, scope: !50) +!58 = !DILocation(line: 10, column: 10, scope: !50) +!59 = !DILocation(line: 9, column: 27, scope: !50) +!60 = distinct !{!60, !54, !61} +!61 = !DILocation(line: 10, column: 12, scope: !45) +!62 = !DILocation(line: 11, column: 1, scope: !41) diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.expected new file mode 100644 index 000000000000..1f9c37ccfbd8 --- /dev/null +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.expected @@ -0,0 +1,238 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; Just run it through opt, no passes needed. +; RUN: opt < %s -S --write-experimental-debuginfo=true | FileCheck %s + +; ModuleID = 'various_ir_values.c' +source_filename = "various_ir_values.c" +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +; Function Attrs: nounwind uwtable +define dso_local void @foo(ptr %A) #0 !dbg !7 { +; CHECK-LABEL: @foo( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[A_ADDR:%.*]] = alloca ptr, align 8, !DIAssignID [[DIASSIGNID16:![0-9]+]] +; CHECK-NEXT: [[I:%.*]] = alloca i32, align 4 +; CHECK-NEXT: #dbg_assign(i1 undef, [[META13:![0-9]+]], !DIExpression(), [[DIASSIGNID16]], ptr [[A_ADDR]], !DIExpression(), [[META17:![0-9]+]]) +; CHECK-NEXT: store ptr [[A:%.*]], ptr [[A_ADDR]], align 8, !tbaa [[TBAA18:![0-9]+]] +; CHECK-NEXT: #dbg_declare(ptr [[A_ADDR]], [[META13]], !DIExpression(), [[META17]]) +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR2:[0-9]+]], !dbg [[DBG22:![0-9]+]] +; CHECK-NEXT: #dbg_declare(ptr [[I]], [[META14:![0-9]+]], !DIExpression(), [[META23:![0-9]+]]) +; CHECK-NEXT: store i32 0, ptr [[I]], align 4, !dbg [[META23]], !tbaa [[TBAA24:![0-9]+]] +; CHECK-NEXT: br label [[FOR_COND:%.*]], !dbg [[DBG22]] +; CHECK: for.cond: +; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[I]], align 4, !dbg [[DBG26:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 8, !dbg [[DBG28:![0-9]+]], !tbaa [[TBAA18]] +; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4, !dbg [[DBG29:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP0]], [[TMP2]], !dbg [[DBG30:![0-9]+]] +; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]], !dbg [[DBG31:![0-9]+]], !prof [[PROF32:![0-9]+]] +; CHECK: for.cond.cleanup: +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR2]], !dbg [[DBG33:![0-9]+]] +; CHECK-NEXT: br label [[FOR_END:%.*]] +; CHECK: for.body: +; CHECK-NEXT: [[TMP3:%.*]] = load ptr, ptr [[A_ADDR]], align 8, !dbg [[DBG34:![0-9]+]], !tbaa [[TBAA18]] +; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[I]], align 4, !dbg [[DBG35:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[IDXPROM:%.*]] = sext i32 [[TMP4]] to i64, !dbg [[DBG34]] +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 [[IDXPROM]], !dbg [[DBG34]] +; CHECK-NEXT: store i32 0, ptr [[ARRAYIDX]], align 4, !dbg [[DBG36:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: br label [[FOR_INC:%.*]], !dbg [[DBG34]] +; CHECK: for.inc: +; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[I]], align 4, !dbg [[DBG37:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP5]], 1, !dbg [[DBG37]] +; CHECK-NEXT: store i32 [[INC]], ptr [[I]], align 4, !dbg [[DBG37]], !tbaa [[TBAA24]] +; CHECK-NEXT: br label [[FOR_COND]], !dbg [[DBG33]], !llvm.loop [[LOOP38:![0-9]+]] +; CHECK: for.end: +; CHECK-NEXT: ret void, !dbg [[DBG40:![0-9]+]] +; +entry: + %A.addr = alloca ptr, align 8, !DIAssignID !16 + %i = alloca i32, align 4 + #dbg_assign(i1 undef, !13, !DIExpression(), !16, ptr %A.addr, !DIExpression(), !17) + store ptr %A, ptr %A.addr, align 8, !tbaa !18 + #dbg_declare(ptr %A.addr, !13, !DIExpression(), !17) + call void @llvm.lifetime.start.p0(i64 4, ptr %i) #2, !dbg !22 + #dbg_declare(ptr %i, !14, !DIExpression(), !23) + store i32 0, ptr %i, align 4, !dbg !23, !tbaa !24 + br label %for.cond, !dbg !22 + +for.cond: ; preds = %for.inc, %entry + %0 = load i32, ptr %i, align 4, !dbg !26, !tbaa !24 + %1 = load ptr, ptr %A.addr, align 8, !dbg !28, !tbaa !18 + %2 = load i32, ptr %1, align 4, !dbg !29, !tbaa !24 + %cmp = icmp slt i32 %0, %2, !dbg !30 + br i1 %cmp, label %for.body, label %for.cond.cleanup, !dbg !31, !prof !32 + +for.cond.cleanup: ; preds = %for.cond + call void @llvm.lifetime.end.p0(i64 4, ptr %i) #2, !dbg !33 + br label %for.end + +for.body: ; preds = %for.cond + %3 = load ptr, ptr %A.addr, align 8, !dbg !34, !tbaa !18 + %4 = load i32, ptr %i, align 4, !dbg !35, !tbaa !24 + %idxprom = sext i32 %4 to i64, !dbg !34 + %arrayidx = getelementptr inbounds i32, ptr %3, i64 %idxprom, !dbg !34 + store i32 0, ptr %arrayidx, align 4, !dbg !36, !tbaa !24 + br label %for.inc, !dbg !34 + +for.inc: ; preds = %for.body + %5 = load i32, ptr %i, align 4, !dbg !37, !tbaa !24 + %inc = add nsw i32 %5, 1, !dbg !37 + store i32 %inc, ptr %i, align 4, !dbg !37, !tbaa !24 + br label %for.cond, !dbg !33, !llvm.loop !38 + +for.end: ; preds = %for.cond.cleanup + ret void, !dbg !40 +} + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) +declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1 + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) +declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1 + +; Function Attrs: nounwind uwtable +define dso_local void @bar(ptr %A) #0 !dbg !41 { +; CHECK-LABEL: @bar( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[A_ADDR:%.*]] = alloca ptr, align 8 +; CHECK-NEXT: [[I:%.*]] = alloca i32, align 4 +; CHECK-NEXT: store ptr [[A:%.*]], ptr [[A_ADDR]], align 8, !tbaa [[TBAA18]] +; CHECK-NEXT: #dbg_declare(ptr [[A_ADDR]], [[META43:![0-9]+]], !DIExpression(), [[META46:![0-9]+]]) +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR2]], !dbg [[DBG47:![0-9]+]] +; CHECK-NEXT: #dbg_declare(ptr [[I]], [[META44:![0-9]+]], !DIExpression(), [[META48:![0-9]+]]) +; CHECK-NEXT: store i32 0, ptr [[I]], align 4, !dbg [[META48]], !tbaa [[TBAA24]] +; CHECK-NEXT: br label [[FOR_COND:%.*]], !dbg [[DBG47]] +; CHECK: for.cond: +; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[I]], align 4, !dbg [[DBG49:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 8, !dbg [[DBG51:![0-9]+]], !tbaa [[TBAA18]] +; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4, !dbg [[DBG52:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP0]], [[TMP2]], !dbg [[DBG53:![0-9]+]] +; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]], !dbg [[DBG54:![0-9]+]] +; CHECK: for.cond.cleanup: +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR2]], !dbg [[DBG55:![0-9]+]] +; CHECK-NEXT: br label [[FOR_END:%.*]] +; CHECK: for.body: +; CHECK-NEXT: [[TMP3:%.*]] = load ptr, ptr [[A_ADDR]], align 8, !dbg [[DBG56:![0-9]+]], !tbaa [[TBAA18]] +; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[I]], align 4, !dbg [[DBG57:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[IDXPROM:%.*]] = sext i32 [[TMP4]] to i64, !dbg [[DBG56]] +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 [[IDXPROM]], !dbg [[DBG56]] +; CHECK-NEXT: store i32 0, ptr [[ARRAYIDX]], align 4, !dbg [[DBG58:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: br label [[FOR_INC:%.*]], !dbg [[DBG56]] +; CHECK: for.inc: +; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[I]], align 4, !dbg [[DBG59:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP5]], 1, !dbg [[DBG59]] +; CHECK-NEXT: store i32 [[INC]], ptr [[I]], align 4, !dbg [[DBG59]], !tbaa [[TBAA24]] +; CHECK-NEXT: br label [[FOR_COND]], !dbg [[DBG55]], !llvm.loop [[LOOP60:![0-9]+]] +; CHECK: for.end: +; CHECK-NEXT: ret void, !dbg [[DBG62:![0-9]+]] +; +entry: + %A.addr = alloca ptr, align 8 + %i = alloca i32, align 4 + store ptr %A, ptr %A.addr, align 8, !tbaa !18 + #dbg_declare(ptr %A.addr, !43, !DIExpression(), !46) + call void @llvm.lifetime.start.p0(i64 4, ptr %i) #2, !dbg !47 + #dbg_declare(ptr %i, !44, !DIExpression(), !48) + store i32 0, ptr %i, align 4, !dbg !48, !tbaa !24 + br label %for.cond, !dbg !47 + +for.cond: ; preds = %for.inc, %entry + %0 = load i32, ptr %i, align 4, !dbg !49, !tbaa !24 + %1 = load ptr, ptr %A.addr, align 8, !dbg !51, !tbaa !18 + %2 = load i32, ptr %1, align 4, !dbg !52, !tbaa !24 + %cmp = icmp slt i32 %0, %2, !dbg !53 + br i1 %cmp, label %for.body, label %for.cond.cleanup, !dbg !54 + +for.cond.cleanup: ; preds = %for.cond + call void @llvm.lifetime.end.p0(i64 4, ptr %i) #2, !dbg !55 + br label %for.end + +for.body: ; preds = %for.cond + %3 = load ptr, ptr %A.addr, align 8, !dbg !56, !tbaa !18 + %4 = load i32, ptr %i, align 4, !dbg !57, !tbaa !24 + %idxprom = sext i32 %4 to i64, !dbg !56 + %arrayidx = getelementptr inbounds i32, ptr %3, i64 %idxprom, !dbg !56 + store i32 0, ptr %arrayidx, align 4, !dbg !58, !tbaa !24 + br label %for.inc, !dbg !56 + +for.inc: ; preds = %for.body + %5 = load i32, ptr %i, align 4, !dbg !59, !tbaa !24 + %inc = add nsw i32 %5, 1, !dbg !59 + store i32 %inc, ptr %i, align 4, !dbg !59, !tbaa !24 + br label %for.cond, !dbg !55, !llvm.loop !60 + +for.end: ; preds = %for.cond.cleanup + ret void, !dbg !62 +} + +attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #1 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) } +attributes #2 = { nounwind } + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!3, !4, !5} +!llvm.ident = !{!6} + +!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 11.0.0 (git@github.com:llvm/llvm-project.git 1d5da8cd30fce1c0a2c2fa6ba656dbfaa36192c8)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, splitDebugInlining: false, nameTableKind: None) +!1 = !DIFile(filename: "various_ir_values.c", directory: "/data/build/llvm-project") +!2 = !{} +!3 = !{i32 7, !"Dwarf Version", i32 4} +!4 = !{i32 2, !"Debug Info Version", i32 3} +!5 = !{i32 1, !"wchar_size", i32 4} +!6 = !{!"clang version 11.0.0 (git@github.com:llvm/llvm-project.git 1d5da8cd30fce1c0a2c2fa6ba656dbfaa36192c8)"} +!7 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !8, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !12) +!8 = !DISubroutineType(types: !9) +!9 = !{null, !10} +!10 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !11, size: 64) +!11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!12 = !{!13, !14} +!13 = !DILocalVariable(name: "A", arg: 1, scope: !7, file: !1, line: 1, type: !10) +!14 = !DILocalVariable(name: "i", scope: !15, file: !1, line: 3, type: !11) +!15 = distinct !DILexicalBlock(scope: !7, file: !1, line: 3, column: 3) +!16 = distinct !DIAssignID() +!17 = !DILocation(line: 1, column: 15, scope: !7) +!18 = !{!19, !19, i64 0} +!19 = !{!"any pointer", !20, i64 0} +!20 = !{!"omnipotent char", !21, i64 0} +!21 = !{!"Simple C/C++ TBAA"} +!22 = !DILocation(line: 3, column: 8, scope: !15) +!23 = !DILocation(line: 3, column: 12, scope: !15) +!24 = !{!25, !25, i64 0} +!25 = !{!"int", !20, i64 0} +!26 = !DILocation(line: 3, column: 19, scope: !27) +!27 = distinct !DILexicalBlock(scope: !15, file: !1, line: 3, column: 3) +!28 = !DILocation(line: 3, column: 24, scope: !27) +!29 = !DILocation(line: 3, column: 23, scope: !27) +!30 = !DILocation(line: 3, column: 21, scope: !27) +!31 = !DILocation(line: 3, column: 3, scope: !15) +!32 = !{!"branch_weights", i32 1, i32 1048575} +!33 = !DILocation(line: 3, column: 3, scope: !27) +!34 = !DILocation(line: 4, column: 5, scope: !27) +!35 = !DILocation(line: 4, column: 7, scope: !27) +!36 = !DILocation(line: 4, column: 10, scope: !27) +!37 = !DILocation(line: 3, column: 27, scope: !27) +!38 = distinct !{!38, !31, !39} +!39 = !DILocation(line: 4, column: 12, scope: !15) +!40 = !DILocation(line: 5, column: 1, scope: !7) +!41 = distinct !DISubprogram(name: "bar", scope: !1, file: !1, line: 7, type: !8, scopeLine: 7, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !42) +!42 = !{!43, !44} +!43 = !DILocalVariable(name: "A", arg: 1, scope: !41, file: !1, line: 7, type: !10) +!44 = !DILocalVariable(name: "i", scope: !45, file: !1, line: 9, type: !11) +!45 = distinct !DILexicalBlock(scope: !41, file: !1, line: 9, column: 3) +!46 = !DILocation(line: 7, column: 15, scope: !41) +!47 = !DILocation(line: 9, column: 8, scope: !45) +!48 = !DILocation(line: 9, column: 12, scope: !45) +!49 = !DILocation(line: 9, column: 19, scope: !50) +!50 = distinct !DILexicalBlock(scope: !45, file: !1, line: 9, column: 3) +!51 = !DILocation(line: 9, column: 24, scope: !50) +!52 = !DILocation(line: 9, column: 23, scope: !50) +!53 = !DILocation(line: 9, column: 21, scope: !50) +!54 = !DILocation(line: 9, column: 3, scope: !45) +!55 = !DILocation(line: 9, column: 3, scope: !50) +!56 = !DILocation(line: 10, column: 5, scope: !50) +!57 = !DILocation(line: 10, column: 7, scope: !50) +!58 = !DILocation(line: 10, column: 10, scope: !50) +!59 = !DILocation(line: 9, column: 27, scope: !50) +!60 = distinct !{!60, !54, !61} +!61 = !DILocation(line: 10, column: 12, scope: !45) +!62 = !DILocation(line: 11, column: 1, scope: !41) diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.funcsig.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.funcsig.expected new file mode 100644 index 000000000000..5905e443deff --- /dev/null +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.funcsig.expected @@ -0,0 +1,240 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature +; Just run it through opt, no passes needed. +; RUN: opt < %s -S --write-experimental-debuginfo=true | FileCheck %s + +; ModuleID = 'various_ir_values.c' +source_filename = "various_ir_values.c" +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +; Function Attrs: nounwind uwtable +define dso_local void @foo(ptr %A) #0 !dbg !7 { +; CHECK-LABEL: define {{[^@]+}}@foo +; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] !dbg [[DBG7:![0-9]+]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[A_ADDR:%.*]] = alloca ptr, align 8, !DIAssignID [[DIASSIGNID16:![0-9]+]] +; CHECK-NEXT: [[I:%.*]] = alloca i32, align 4 +; CHECK-NEXT: #dbg_assign(i1 undef, [[META13:![0-9]+]], !DIExpression(), [[DIASSIGNID16]], ptr [[A_ADDR]], !DIExpression(), [[META17:![0-9]+]]) +; CHECK-NEXT: store ptr [[A]], ptr [[A_ADDR]], align 8, !tbaa [[TBAA18:![0-9]+]] +; CHECK-NEXT: #dbg_declare(ptr [[A_ADDR]], [[META13]], !DIExpression(), [[META17]]) +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR2:[0-9]+]], !dbg [[DBG22:![0-9]+]] +; CHECK-NEXT: #dbg_declare(ptr [[I]], [[META14:![0-9]+]], !DIExpression(), [[META23:![0-9]+]]) +; CHECK-NEXT: store i32 0, ptr [[I]], align 4, !dbg [[META23]], !tbaa [[TBAA24:![0-9]+]] +; CHECK-NEXT: br label [[FOR_COND:%.*]], !dbg [[DBG22]] +; CHECK: for.cond: +; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[I]], align 4, !dbg [[DBG26:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 8, !dbg [[DBG28:![0-9]+]], !tbaa [[TBAA18]] +; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4, !dbg [[DBG29:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP0]], [[TMP2]], !dbg [[DBG30:![0-9]+]] +; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]], !dbg [[DBG31:![0-9]+]], !prof [[PROF32:![0-9]+]] +; CHECK: for.cond.cleanup: +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR2]], !dbg [[DBG33:![0-9]+]] +; CHECK-NEXT: br label [[FOR_END:%.*]] +; CHECK: for.body: +; CHECK-NEXT: [[TMP3:%.*]] = load ptr, ptr [[A_ADDR]], align 8, !dbg [[DBG34:![0-9]+]], !tbaa [[TBAA18]] +; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[I]], align 4, !dbg [[DBG35:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[IDXPROM:%.*]] = sext i32 [[TMP4]] to i64, !dbg [[DBG34]] +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 [[IDXPROM]], !dbg [[DBG34]] +; CHECK-NEXT: store i32 0, ptr [[ARRAYIDX]], align 4, !dbg [[DBG36:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: br label [[FOR_INC:%.*]], !dbg [[DBG34]] +; CHECK: for.inc: +; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[I]], align 4, !dbg [[DBG37:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP5]], 1, !dbg [[DBG37]] +; CHECK-NEXT: store i32 [[INC]], ptr [[I]], align 4, !dbg [[DBG37]], !tbaa [[TBAA24]] +; CHECK-NEXT: br label [[FOR_COND]], !dbg [[DBG33]], !llvm.loop [[LOOP38:![0-9]+]] +; CHECK: for.end: +; CHECK-NEXT: ret void, !dbg [[DBG40:![0-9]+]] +; +entry: + %A.addr = alloca ptr, align 8, !DIAssignID !16 + %i = alloca i32, align 4 + #dbg_assign(i1 undef, !13, !DIExpression(), !16, ptr %A.addr, !DIExpression(), !17) + store ptr %A, ptr %A.addr, align 8, !tbaa !18 + #dbg_declare(ptr %A.addr, !13, !DIExpression(), !17) + call void @llvm.lifetime.start.p0(i64 4, ptr %i) #2, !dbg !22 + #dbg_declare(ptr %i, !14, !DIExpression(), !23) + store i32 0, ptr %i, align 4, !dbg !23, !tbaa !24 + br label %for.cond, !dbg !22 + +for.cond: ; preds = %for.inc, %entry + %0 = load i32, ptr %i, align 4, !dbg !26, !tbaa !24 + %1 = load ptr, ptr %A.addr, align 8, !dbg !28, !tbaa !18 + %2 = load i32, ptr %1, align 4, !dbg !29, !tbaa !24 + %cmp = icmp slt i32 %0, %2, !dbg !30 + br i1 %cmp, label %for.body, label %for.cond.cleanup, !dbg !31, !prof !32 + +for.cond.cleanup: ; preds = %for.cond + call void @llvm.lifetime.end.p0(i64 4, ptr %i) #2, !dbg !33 + br label %for.end + +for.body: ; preds = %for.cond + %3 = load ptr, ptr %A.addr, align 8, !dbg !34, !tbaa !18 + %4 = load i32, ptr %i, align 4, !dbg !35, !tbaa !24 + %idxprom = sext i32 %4 to i64, !dbg !34 + %arrayidx = getelementptr inbounds i32, ptr %3, i64 %idxprom, !dbg !34 + store i32 0, ptr %arrayidx, align 4, !dbg !36, !tbaa !24 + br label %for.inc, !dbg !34 + +for.inc: ; preds = %for.body + %5 = load i32, ptr %i, align 4, !dbg !37, !tbaa !24 + %inc = add nsw i32 %5, 1, !dbg !37 + store i32 %inc, ptr %i, align 4, !dbg !37, !tbaa !24 + br label %for.cond, !dbg !33, !llvm.loop !38 + +for.end: ; preds = %for.cond.cleanup + ret void, !dbg !40 +} + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) +declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1 + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) +declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1 + +; Function Attrs: nounwind uwtable +define dso_local void @bar(ptr %A) #0 !dbg !41 { +; CHECK-LABEL: define {{[^@]+}}@bar +; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0]] !dbg [[DBG41:![0-9]+]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[A_ADDR:%.*]] = alloca ptr, align 8 +; CHECK-NEXT: [[I:%.*]] = alloca i32, align 4 +; CHECK-NEXT: store ptr [[A]], ptr [[A_ADDR]], align 8, !tbaa [[TBAA18]] +; CHECK-NEXT: #dbg_declare(ptr [[A_ADDR]], [[META43:![0-9]+]], !DIExpression(), [[META46:![0-9]+]]) +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR2]], !dbg [[DBG47:![0-9]+]] +; CHECK-NEXT: #dbg_declare(ptr [[I]], [[META44:![0-9]+]], !DIExpression(), [[META48:![0-9]+]]) +; CHECK-NEXT: store i32 0, ptr [[I]], align 4, !dbg [[META48]], !tbaa [[TBAA24]] +; CHECK-NEXT: br label [[FOR_COND:%.*]], !dbg [[DBG47]] +; CHECK: for.cond: +; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[I]], align 4, !dbg [[DBG49:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 8, !dbg [[DBG51:![0-9]+]], !tbaa [[TBAA18]] +; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4, !dbg [[DBG52:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP0]], [[TMP2]], !dbg [[DBG53:![0-9]+]] +; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]], !dbg [[DBG54:![0-9]+]] +; CHECK: for.cond.cleanup: +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR2]], !dbg [[DBG55:![0-9]+]] +; CHECK-NEXT: br label [[FOR_END:%.*]] +; CHECK: for.body: +; CHECK-NEXT: [[TMP3:%.*]] = load ptr, ptr [[A_ADDR]], align 8, !dbg [[DBG56:![0-9]+]], !tbaa [[TBAA18]] +; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[I]], align 4, !dbg [[DBG57:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[IDXPROM:%.*]] = sext i32 [[TMP4]] to i64, !dbg [[DBG56]] +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 [[IDXPROM]], !dbg [[DBG56]] +; CHECK-NEXT: store i32 0, ptr [[ARRAYIDX]], align 4, !dbg [[DBG58:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: br label [[FOR_INC:%.*]], !dbg [[DBG56]] +; CHECK: for.inc: +; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[I]], align 4, !dbg [[DBG59:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP5]], 1, !dbg [[DBG59]] +; CHECK-NEXT: store i32 [[INC]], ptr [[I]], align 4, !dbg [[DBG59]], !tbaa [[TBAA24]] +; CHECK-NEXT: br label [[FOR_COND]], !dbg [[DBG55]], !llvm.loop [[LOOP60:![0-9]+]] +; CHECK: for.end: +; CHECK-NEXT: ret void, !dbg [[DBG62:![0-9]+]] +; +entry: + %A.addr = alloca ptr, align 8 + %i = alloca i32, align 4 + store ptr %A, ptr %A.addr, align 8, !tbaa !18 + #dbg_declare(ptr %A.addr, !43, !DIExpression(), !46) + call void @llvm.lifetime.start.p0(i64 4, ptr %i) #2, !dbg !47 + #dbg_declare(ptr %i, !44, !DIExpression(), !48) + store i32 0, ptr %i, align 4, !dbg !48, !tbaa !24 + br label %for.cond, !dbg !47 + +for.cond: ; preds = %for.inc, %entry + %0 = load i32, ptr %i, align 4, !dbg !49, !tbaa !24 + %1 = load ptr, ptr %A.addr, align 8, !dbg !51, !tbaa !18 + %2 = load i32, ptr %1, align 4, !dbg !52, !tbaa !24 + %cmp = icmp slt i32 %0, %2, !dbg !53 + br i1 %cmp, label %for.body, label %for.cond.cleanup, !dbg !54 + +for.cond.cleanup: ; preds = %for.cond + call void @llvm.lifetime.end.p0(i64 4, ptr %i) #2, !dbg !55 + br label %for.end + +for.body: ; preds = %for.cond + %3 = load ptr, ptr %A.addr, align 8, !dbg !56, !tbaa !18 + %4 = load i32, ptr %i, align 4, !dbg !57, !tbaa !24 + %idxprom = sext i32 %4 to i64, !dbg !56 + %arrayidx = getelementptr inbounds i32, ptr %3, i64 %idxprom, !dbg !56 + store i32 0, ptr %arrayidx, align 4, !dbg !58, !tbaa !24 + br label %for.inc, !dbg !56 + +for.inc: ; preds = %for.body + %5 = load i32, ptr %i, align 4, !dbg !59, !tbaa !24 + %inc = add nsw i32 %5, 1, !dbg !59 + store i32 %inc, ptr %i, align 4, !dbg !59, !tbaa !24 + br label %for.cond, !dbg !55, !llvm.loop !60 + +for.end: ; preds = %for.cond.cleanup + ret void, !dbg !62 +} + +attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #1 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) } +attributes #2 = { nounwind } + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!3, !4, !5} +!llvm.ident = !{!6} + +!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 11.0.0 (git@github.com:llvm/llvm-project.git 1d5da8cd30fce1c0a2c2fa6ba656dbfaa36192c8)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, splitDebugInlining: false, nameTableKind: None) +!1 = !DIFile(filename: "various_ir_values.c", directory: "/data/build/llvm-project") +!2 = !{} +!3 = !{i32 7, !"Dwarf Version", i32 4} +!4 = !{i32 2, !"Debug Info Version", i32 3} +!5 = !{i32 1, !"wchar_size", i32 4} +!6 = !{!"clang version 11.0.0 (git@github.com:llvm/llvm-project.git 1d5da8cd30fce1c0a2c2fa6ba656dbfaa36192c8)"} +!7 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !8, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !12) +!8 = !DISubroutineType(types: !9) +!9 = !{null, !10} +!10 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !11, size: 64) +!11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!12 = !{!13, !14} +!13 = !DILocalVariable(name: "A", arg: 1, scope: !7, file: !1, line: 1, type: !10) +!14 = !DILocalVariable(name: "i", scope: !15, file: !1, line: 3, type: !11) +!15 = distinct !DILexicalBlock(scope: !7, file: !1, line: 3, column: 3) +!16 = distinct !DIAssignID() +!17 = !DILocation(line: 1, column: 15, scope: !7) +!18 = !{!19, !19, i64 0} +!19 = !{!"any pointer", !20, i64 0} +!20 = !{!"omnipotent char", !21, i64 0} +!21 = !{!"Simple C/C++ TBAA"} +!22 = !DILocation(line: 3, column: 8, scope: !15) +!23 = !DILocation(line: 3, column: 12, scope: !15) +!24 = !{!25, !25, i64 0} +!25 = !{!"int", !20, i64 0} +!26 = !DILocation(line: 3, column: 19, scope: !27) +!27 = distinct !DILexicalBlock(scope: !15, file: !1, line: 3, column: 3) +!28 = !DILocation(line: 3, column: 24, scope: !27) +!29 = !DILocation(line: 3, column: 23, scope: !27) +!30 = !DILocation(line: 3, column: 21, scope: !27) +!31 = !DILocation(line: 3, column: 3, scope: !15) +!32 = !{!"branch_weights", i32 1, i32 1048575} +!33 = !DILocation(line: 3, column: 3, scope: !27) +!34 = !DILocation(line: 4, column: 5, scope: !27) +!35 = !DILocation(line: 4, column: 7, scope: !27) +!36 = !DILocation(line: 4, column: 10, scope: !27) +!37 = !DILocation(line: 3, column: 27, scope: !27) +!38 = distinct !{!38, !31, !39} +!39 = !DILocation(line: 4, column: 12, scope: !15) +!40 = !DILocation(line: 5, column: 1, scope: !7) +!41 = distinct !DISubprogram(name: "bar", scope: !1, file: !1, line: 7, type: !8, scopeLine: 7, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !42) +!42 = !{!43, !44} +!43 = !DILocalVariable(name: "A", arg: 1, scope: !41, file: !1, line: 7, type: !10) +!44 = !DILocalVariable(name: "i", scope: !45, file: !1, line: 9, type: !11) +!45 = distinct !DILexicalBlock(scope: !41, file: !1, line: 9, column: 3) +!46 = !DILocation(line: 7, column: 15, scope: !41) +!47 = !DILocation(line: 9, column: 8, scope: !45) +!48 = !DILocation(line: 9, column: 12, scope: !45) +!49 = !DILocation(line: 9, column: 19, scope: !50) +!50 = distinct !DILexicalBlock(scope: !45, file: !1, line: 9, column: 3) +!51 = !DILocation(line: 9, column: 24, scope: !50) +!52 = !DILocation(line: 9, column: 23, scope: !50) +!53 = !DILocation(line: 9, column: 21, scope: !50) +!54 = !DILocation(line: 9, column: 3, scope: !45) +!55 = !DILocation(line: 9, column: 3, scope: !50) +!56 = !DILocation(line: 10, column: 5, scope: !50) +!57 = !DILocation(line: 10, column: 7, scope: !50) +!58 = !DILocation(line: 10, column: 10, scope: !50) +!59 = !DILocation(line: 9, column: 27, scope: !50) +!60 = distinct !{!60, !54, !61} +!61 = !DILocation(line: 10, column: 12, scope: !45) +!62 = !DILocation(line: 11, column: 1, scope: !41) diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.funcsig.globals.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.funcsig.globals.expected new file mode 100644 index 000000000000..579d6a437d0e --- /dev/null +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.funcsig.globals.expected @@ -0,0 +1,309 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --check-globals +; Just run it through opt, no passes needed. +; RUN: opt < %s -S --write-experimental-debuginfo=true | FileCheck %s + +; ModuleID = 'various_ir_values.c' +source_filename = "various_ir_values.c" +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +; Function Attrs: nounwind uwtable +define dso_local void @foo(ptr %A) #0 !dbg !7 { +; CHECK-LABEL: define {{[^@]+}}@foo +; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] !dbg [[DBG7:![0-9]+]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[A_ADDR:%.*]] = alloca ptr, align 8, !DIAssignID [[DIASSIGNID16:![0-9]+]] +; CHECK-NEXT: [[I:%.*]] = alloca i32, align 4 +; CHECK-NEXT: #dbg_assign(i1 undef, [[META13:![0-9]+]], !DIExpression(), [[DIASSIGNID16]], ptr [[A_ADDR]], !DIExpression(), [[META17:![0-9]+]]) +; CHECK-NEXT: store ptr [[A]], ptr [[A_ADDR]], align 8, !tbaa [[TBAA18:![0-9]+]] +; CHECK-NEXT: #dbg_declare(ptr [[A_ADDR]], [[META13]], !DIExpression(), [[META17]]) +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR2:[0-9]+]], !dbg [[DBG22:![0-9]+]] +; CHECK-NEXT: #dbg_declare(ptr [[I]], [[META14:![0-9]+]], !DIExpression(), [[META23:![0-9]+]]) +; CHECK-NEXT: store i32 0, ptr [[I]], align 4, !dbg [[META23]], !tbaa [[TBAA24:![0-9]+]] +; CHECK-NEXT: br label [[FOR_COND:%.*]], !dbg [[DBG22]] +; CHECK: for.cond: +; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[I]], align 4, !dbg [[DBG26:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 8, !dbg [[DBG28:![0-9]+]], !tbaa [[TBAA18]] +; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4, !dbg [[DBG29:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP0]], [[TMP2]], !dbg [[DBG30:![0-9]+]] +; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]], !dbg [[DBG31:![0-9]+]], !prof [[PROF32:![0-9]+]] +; CHECK: for.cond.cleanup: +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR2]], !dbg [[DBG33:![0-9]+]] +; CHECK-NEXT: br label [[FOR_END:%.*]] +; CHECK: for.body: +; CHECK-NEXT: [[TMP3:%.*]] = load ptr, ptr [[A_ADDR]], align 8, !dbg [[DBG34:![0-9]+]], !tbaa [[TBAA18]] +; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[I]], align 4, !dbg [[DBG35:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[IDXPROM:%.*]] = sext i32 [[TMP4]] to i64, !dbg [[DBG34]] +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 [[IDXPROM]], !dbg [[DBG34]] +; CHECK-NEXT: store i32 0, ptr [[ARRAYIDX]], align 4, !dbg [[DBG36:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: br label [[FOR_INC:%.*]], !dbg [[DBG34]] +; CHECK: for.inc: +; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[I]], align 4, !dbg [[DBG37:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP5]], 1, !dbg [[DBG37]] +; CHECK-NEXT: store i32 [[INC]], ptr [[I]], align 4, !dbg [[DBG37]], !tbaa [[TBAA24]] +; CHECK-NEXT: br label [[FOR_COND]], !dbg [[DBG33]], !llvm.loop [[LOOP38:![0-9]+]] +; CHECK: for.end: +; CHECK-NEXT: ret void, !dbg [[DBG40:![0-9]+]] +; +entry: + %A.addr = alloca ptr, align 8, !DIAssignID !16 + %i = alloca i32, align 4 + #dbg_assign(i1 undef, !13, !DIExpression(), !16, ptr %A.addr, !DIExpression(), !17) + store ptr %A, ptr %A.addr, align 8, !tbaa !18 + #dbg_declare(ptr %A.addr, !13, !DIExpression(), !17) + call void @llvm.lifetime.start.p0(i64 4, ptr %i) #2, !dbg !22 + #dbg_declare(ptr %i, !14, !DIExpression(), !23) + store i32 0, ptr %i, align 4, !dbg !23, !tbaa !24 + br label %for.cond, !dbg !22 + +for.cond: ; preds = %for.inc, %entry + %0 = load i32, ptr %i, align 4, !dbg !26, !tbaa !24 + %1 = load ptr, ptr %A.addr, align 8, !dbg !28, !tbaa !18 + %2 = load i32, ptr %1, align 4, !dbg !29, !tbaa !24 + %cmp = icmp slt i32 %0, %2, !dbg !30 + br i1 %cmp, label %for.body, label %for.cond.cleanup, !dbg !31, !prof !32 + +for.cond.cleanup: ; preds = %for.cond + call void @llvm.lifetime.end.p0(i64 4, ptr %i) #2, !dbg !33 + br label %for.end + +for.body: ; preds = %for.cond + %3 = load ptr, ptr %A.addr, align 8, !dbg !34, !tbaa !18 + %4 = load i32, ptr %i, align 4, !dbg !35, !tbaa !24 + %idxprom = sext i32 %4 to i64, !dbg !34 + %arrayidx = getelementptr inbounds i32, ptr %3, i64 %idxprom, !dbg !34 + store i32 0, ptr %arrayidx, align 4, !dbg !36, !tbaa !24 + br label %for.inc, !dbg !34 + +for.inc: ; preds = %for.body + %5 = load i32, ptr %i, align 4, !dbg !37, !tbaa !24 + %inc = add nsw i32 %5, 1, !dbg !37 + store i32 %inc, ptr %i, align 4, !dbg !37, !tbaa !24 + br label %for.cond, !dbg !33, !llvm.loop !38 + +for.end: ; preds = %for.cond.cleanup + ret void, !dbg !40 +} + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) +declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1 + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) +declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1 + +; Function Attrs: nounwind uwtable +define dso_local void @bar(ptr %A) #0 !dbg !41 { +; CHECK-LABEL: define {{[^@]+}}@bar +; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0]] !dbg [[DBG41:![0-9]+]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[A_ADDR:%.*]] = alloca ptr, align 8 +; CHECK-NEXT: [[I:%.*]] = alloca i32, align 4 +; CHECK-NEXT: store ptr [[A]], ptr [[A_ADDR]], align 8, !tbaa [[TBAA18]] +; CHECK-NEXT: #dbg_declare(ptr [[A_ADDR]], [[META43:![0-9]+]], !DIExpression(), [[META46:![0-9]+]]) +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR2]], !dbg [[DBG47:![0-9]+]] +; CHECK-NEXT: #dbg_declare(ptr [[I]], [[META44:![0-9]+]], !DIExpression(), [[META48:![0-9]+]]) +; CHECK-NEXT: store i32 0, ptr [[I]], align 4, !dbg [[META48]], !tbaa [[TBAA24]] +; CHECK-NEXT: br label [[FOR_COND:%.*]], !dbg [[DBG47]] +; CHECK: for.cond: +; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[I]], align 4, !dbg [[DBG49:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 8, !dbg [[DBG51:![0-9]+]], !tbaa [[TBAA18]] +; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4, !dbg [[DBG52:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP0]], [[TMP2]], !dbg [[DBG53:![0-9]+]] +; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]], !dbg [[DBG54:![0-9]+]] +; CHECK: for.cond.cleanup: +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR2]], !dbg [[DBG55:![0-9]+]] +; CHECK-NEXT: br label [[FOR_END:%.*]] +; CHECK: for.body: +; CHECK-NEXT: [[TMP3:%.*]] = load ptr, ptr [[A_ADDR]], align 8, !dbg [[DBG56:![0-9]+]], !tbaa [[TBAA18]] +; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[I]], align 4, !dbg [[DBG57:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[IDXPROM:%.*]] = sext i32 [[TMP4]] to i64, !dbg [[DBG56]] +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 [[IDXPROM]], !dbg [[DBG56]] +; CHECK-NEXT: store i32 0, ptr [[ARRAYIDX]], align 4, !dbg [[DBG58:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: br label [[FOR_INC:%.*]], !dbg [[DBG56]] +; CHECK: for.inc: +; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[I]], align 4, !dbg [[DBG59:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP5]], 1, !dbg [[DBG59]] +; CHECK-NEXT: store i32 [[INC]], ptr [[I]], align 4, !dbg [[DBG59]], !tbaa [[TBAA24]] +; CHECK-NEXT: br label [[FOR_COND]], !dbg [[DBG55]], !llvm.loop [[LOOP60:![0-9]+]] +; CHECK: for.end: +; CHECK-NEXT: ret void, !dbg [[DBG62:![0-9]+]] +; +entry: + %A.addr = alloca ptr, align 8 + %i = alloca i32, align 4 + store ptr %A, ptr %A.addr, align 8, !tbaa !18 + #dbg_declare(ptr %A.addr, !43, !DIExpression(), !46) + call void @llvm.lifetime.start.p0(i64 4, ptr %i) #2, !dbg !47 + #dbg_declare(ptr %i, !44, !DIExpression(), !48) + store i32 0, ptr %i, align 4, !dbg !48, !tbaa !24 + br label %for.cond, !dbg !47 + +for.cond: ; preds = %for.inc, %entry + %0 = load i32, ptr %i, align 4, !dbg !49, !tbaa !24 + %1 = load ptr, ptr %A.addr, align 8, !dbg !51, !tbaa !18 + %2 = load i32, ptr %1, align 4, !dbg !52, !tbaa !24 + %cmp = icmp slt i32 %0, %2, !dbg !53 + br i1 %cmp, label %for.body, label %for.cond.cleanup, !dbg !54 + +for.cond.cleanup: ; preds = %for.cond + call void @llvm.lifetime.end.p0(i64 4, ptr %i) #2, !dbg !55 + br label %for.end + +for.body: ; preds = %for.cond + %3 = load ptr, ptr %A.addr, align 8, !dbg !56, !tbaa !18 + %4 = load i32, ptr %i, align 4, !dbg !57, !tbaa !24 + %idxprom = sext i32 %4 to i64, !dbg !56 + %arrayidx = getelementptr inbounds i32, ptr %3, i64 %idxprom, !dbg !56 + store i32 0, ptr %arrayidx, align 4, !dbg !58, !tbaa !24 + br label %for.inc, !dbg !56 + +for.inc: ; preds = %for.body + %5 = load i32, ptr %i, align 4, !dbg !59, !tbaa !24 + %inc = add nsw i32 %5, 1, !dbg !59 + store i32 %inc, ptr %i, align 4, !dbg !59, !tbaa !24 + br label %for.cond, !dbg !55, !llvm.loop !60 + +for.end: ; preds = %for.cond.cleanup + ret void, !dbg !62 +} + +attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #1 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) } +attributes #2 = { nounwind } + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!3, !4, !5} +!llvm.ident = !{!6} + +!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 11.0.0 (git@github.com:llvm/llvm-project.git 1d5da8cd30fce1c0a2c2fa6ba656dbfaa36192c8)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, splitDebugInlining: false, nameTableKind: None) +!1 = !DIFile(filename: "various_ir_values.c", directory: "/data/build/llvm-project") +!2 = !{} +!3 = !{i32 7, !"Dwarf Version", i32 4} +!4 = !{i32 2, !"Debug Info Version", i32 3} +!5 = !{i32 1, !"wchar_size", i32 4} +!6 = !{!"clang version 11.0.0 (git@github.com:llvm/llvm-project.git 1d5da8cd30fce1c0a2c2fa6ba656dbfaa36192c8)"} +!7 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !8, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !12) +!8 = !DISubroutineType(types: !9) +!9 = !{null, !10} +!10 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !11, size: 64) +!11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!12 = !{!13, !14} +!13 = !DILocalVariable(name: "A", arg: 1, scope: !7, file: !1, line: 1, type: !10) +!14 = !DILocalVariable(name: "i", scope: !15, file: !1, line: 3, type: !11) +!15 = distinct !DILexicalBlock(scope: !7, file: !1, line: 3, column: 3) +!16 = distinct !DIAssignID() +!17 = !DILocation(line: 1, column: 15, scope: !7) +!18 = !{!19, !19, i64 0} +!19 = !{!"any pointer", !20, i64 0} +!20 = !{!"omnipotent char", !21, i64 0} +!21 = !{!"Simple C/C++ TBAA"} +!22 = !DILocation(line: 3, column: 8, scope: !15) +!23 = !DILocation(line: 3, column: 12, scope: !15) +!24 = !{!25, !25, i64 0} +!25 = !{!"int", !20, i64 0} +!26 = !DILocation(line: 3, column: 19, scope: !27) +!27 = distinct !DILexicalBlock(scope: !15, file: !1, line: 3, column: 3) +!28 = !DILocation(line: 3, column: 24, scope: !27) +!29 = !DILocation(line: 3, column: 23, scope: !27) +!30 = !DILocation(line: 3, column: 21, scope: !27) +!31 = !DILocation(line: 3, column: 3, scope: !15) +!32 = !{!"branch_weights", i32 1, i32 1048575} +!33 = !DILocation(line: 3, column: 3, scope: !27) +!34 = !DILocation(line: 4, column: 5, scope: !27) +!35 = !DILocation(line: 4, column: 7, scope: !27) +!36 = !DILocation(line: 4, column: 10, scope: !27) +!37 = !DILocation(line: 3, column: 27, scope: !27) +!38 = distinct !{!38, !31, !39} +!39 = !DILocation(line: 4, column: 12, scope: !15) +!40 = !DILocation(line: 5, column: 1, scope: !7) +!41 = distinct !DISubprogram(name: "bar", scope: !1, file: !1, line: 7, type: !8, scopeLine: 7, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !42) +!42 = !{!43, !44} +!43 = !DILocalVariable(name: "A", arg: 1, scope: !41, file: !1, line: 7, type: !10) +!44 = !DILocalVariable(name: "i", scope: !45, file: !1, line: 9, type: !11) +!45 = distinct !DILexicalBlock(scope: !41, file: !1, line: 9, column: 3) +!46 = !DILocation(line: 7, column: 15, scope: !41) +!47 = !DILocation(line: 9, column: 8, scope: !45) +!48 = !DILocation(line: 9, column: 12, scope: !45) +!49 = !DILocation(line: 9, column: 19, scope: !50) +!50 = distinct !DILexicalBlock(scope: !45, file: !1, line: 9, column: 3) +!51 = !DILocation(line: 9, column: 24, scope: !50) +!52 = !DILocation(line: 9, column: 23, scope: !50) +!53 = !DILocation(line: 9, column: 21, scope: !50) +!54 = !DILocation(line: 9, column: 3, scope: !45) +!55 = !DILocation(line: 9, column: 3, scope: !50) +!56 = !DILocation(line: 10, column: 5, scope: !50) +!57 = !DILocation(line: 10, column: 7, scope: !50) +!58 = !DILocation(line: 10, column: 10, scope: !50) +!59 = !DILocation(line: 9, column: 27, scope: !50) +!60 = distinct !{!60, !54, !61} +!61 = !DILocation(line: 10, column: 12, scope: !45) +!62 = !DILocation(line: 11, column: 1, scope: !41) +;. +; CHECK: attributes #[[ATTR0]] = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } +; CHECK: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) } +; CHECK: attributes #[[ATTR2]] = { nounwind } +;. +; CHECK: [[META0:![0-9]+]] = distinct !DICompileUnit(language: DW_LANG_C99, file: [[META1:![0-9]+]], producer: "{{.*}}clang version {{.*}}", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: [[META2:![0-9]+]], splitDebugInlining: false, nameTableKind: None) +; CHECK: [[META1]] = !DIFile(filename: "various_ir_values.c", directory: {{.*}}) +; CHECK: [[META2]] = !{} +; CHECK: [[META3:![0-9]+]] = !{i32 7, !"Dwarf Version", i32 4} +; CHECK: [[META4:![0-9]+]] = !{i32 2, !"Debug Info Version", i32 3} +; CHECK: [[META5:![0-9]+]] = !{i32 1, !"wchar_size", i32 4} +; CHECK: [[META6:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"} +; CHECK: [[DBG7]] = distinct !DISubprogram(name: "foo", scope: [[META1]], file: [[META1]], line: 1, type: [[META8:![0-9]+]], scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META12:![0-9]+]]) +; CHECK: [[META8]] = !DISubroutineType(types: [[META9:![0-9]+]]) +; CHECK: [[META9]] = !{null, [[META10:![0-9]+]]} +; CHECK: [[META10]] = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: [[META11:![0-9]+]], size: 64) +; CHECK: [[META11]] = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +; CHECK: [[META12]] = !{[[META13]], [[META14]]} +; CHECK: [[META13]] = !DILocalVariable(name: "A", arg: 1, scope: [[DBG7]], file: [[META1]], line: 1, type: [[META10]]) +; CHECK: [[META14]] = !DILocalVariable(name: "i", scope: [[META15:![0-9]+]], file: [[META1]], line: 3, type: [[META11]]) +; CHECK: [[META15]] = distinct !DILexicalBlock(scope: [[DBG7]], file: [[META1]], line: 3, column: 3) +; CHECK: [[DIASSIGNID16]] = distinct !DIAssignID() +; CHECK: [[META17]] = !DILocation(line: 1, column: 15, scope: [[DBG7]]) +; CHECK: [[TBAA18]] = !{[[META19:![0-9]+]], [[META19]], i64 0} +; CHECK: [[META19]] = !{!"any pointer", [[META20:![0-9]+]], i64 0} +; CHECK: [[META20]] = !{!"omnipotent char", [[META21:![0-9]+]], i64 0} +; CHECK: [[META21]] = !{!"Simple C/C++ TBAA"} +; CHECK: [[DBG22]] = !DILocation(line: 3, column: 8, scope: [[META15]]) +; CHECK: [[META23]] = !DILocation(line: 3, column: 12, scope: [[META15]]) +; CHECK: [[TBAA24]] = !{[[META25:![0-9]+]], [[META25]], i64 0} +; CHECK: [[META25]] = !{!"int", [[META20]], i64 0} +; CHECK: [[DBG26]] = !DILocation(line: 3, column: 19, scope: [[META27:![0-9]+]]) +; CHECK: [[META27]] = distinct !DILexicalBlock(scope: [[META15]], file: [[META1]], line: 3, column: 3) +; CHECK: [[DBG28]] = !DILocation(line: 3, column: 24, scope: [[META27]]) +; CHECK: [[DBG29]] = !DILocation(line: 3, column: 23, scope: [[META27]]) +; CHECK: [[DBG30]] = !DILocation(line: 3, column: 21, scope: [[META27]]) +; CHECK: [[DBG31]] = !DILocation(line: 3, column: 3, scope: [[META15]]) +; CHECK: [[PROF32]] = !{!"branch_weights", i32 1, i32 1048575} +; CHECK: [[DBG33]] = !DILocation(line: 3, column: 3, scope: [[META27]]) +; CHECK: [[DBG34]] = !DILocation(line: 4, column: 5, scope: [[META27]]) +; CHECK: [[DBG35]] = !DILocation(line: 4, column: 7, scope: [[META27]]) +; CHECK: [[DBG36]] = !DILocation(line: 4, column: 10, scope: [[META27]]) +; CHECK: [[DBG37]] = !DILocation(line: 3, column: 27, scope: [[META27]]) +; CHECK: [[LOOP38]] = distinct !{[[LOOP38]], [[DBG31]], [[META39:![0-9]+]]} +; CHECK: [[META39]] = !DILocation(line: 4, column: 12, scope: [[META15]]) +; CHECK: [[DBG40]] = !DILocation(line: 5, column: 1, scope: [[DBG7]]) +; CHECK: [[DBG41]] = distinct !DISubprogram(name: "bar", scope: [[META1]], file: [[META1]], line: 7, type: [[META8]], scopeLine: 7, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META42:![0-9]+]]) +; CHECK: [[META42]] = !{[[META43]], [[META44]]} +; CHECK: [[META43]] = !DILocalVariable(name: "A", arg: 1, scope: [[DBG41]], file: [[META1]], line: 7, type: [[META10]]) +; CHECK: [[META44]] = !DILocalVariable(name: "i", scope: [[META45:![0-9]+]], file: [[META1]], line: 9, type: [[META11]]) +; CHECK: [[META45]] = distinct !DILexicalBlock(scope: [[DBG41]], file: [[META1]], line: 9, column: 3) +; CHECK: [[META46]] = !DILocation(line: 7, column: 15, scope: [[DBG41]]) +; CHECK: [[DBG47]] = !DILocation(line: 9, column: 8, scope: [[META45]]) +; CHECK: [[META48]] = !DILocation(line: 9, column: 12, scope: [[META45]]) +; CHECK: [[DBG49]] = !DILocation(line: 9, column: 19, scope: [[META50:![0-9]+]]) +; CHECK: [[META50]] = distinct !DILexicalBlock(scope: [[META45]], file: [[META1]], line: 9, column: 3) +; CHECK: [[DBG51]] = !DILocation(line: 9, column: 24, scope: [[META50]]) +; CHECK: [[DBG52]] = !DILocation(line: 9, column: 23, scope: [[META50]]) +; CHECK: [[DBG53]] = !DILocation(line: 9, column: 21, scope: [[META50]]) +; CHECK: [[DBG54]] = !DILocation(line: 9, column: 3, scope: [[META45]]) +; CHECK: [[DBG55]] = !DILocation(line: 9, column: 3, scope: [[META50]]) +; CHECK: [[DBG56]] = !DILocation(line: 10, column: 5, scope: [[META50]]) +; CHECK: [[DBG57]] = !DILocation(line: 10, column: 7, scope: [[META50]]) +; CHECK: [[DBG58]] = !DILocation(line: 10, column: 10, scope: [[META50]]) +; CHECK: [[DBG59]] = !DILocation(line: 9, column: 27, scope: [[META50]]) +; CHECK: [[LOOP60]] = distinct !{[[LOOP60]], [[DBG54]], [[META61:![0-9]+]]} +; CHECK: [[META61]] = !DILocation(line: 10, column: 12, scope: [[META45]]) +; CHECK: [[DBG62]] = !DILocation(line: 11, column: 1, scope: [[DBG41]]) +;. diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.funcsig.noglobals.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.funcsig.noglobals.expected new file mode 100644 index 000000000000..1f9c37ccfbd8 --- /dev/null +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.funcsig.noglobals.expected @@ -0,0 +1,238 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; Just run it through opt, no passes needed. +; RUN: opt < %s -S --write-experimental-debuginfo=true | FileCheck %s + +; ModuleID = 'various_ir_values.c' +source_filename = "various_ir_values.c" +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +; Function Attrs: nounwind uwtable +define dso_local void @foo(ptr %A) #0 !dbg !7 { +; CHECK-LABEL: @foo( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[A_ADDR:%.*]] = alloca ptr, align 8, !DIAssignID [[DIASSIGNID16:![0-9]+]] +; CHECK-NEXT: [[I:%.*]] = alloca i32, align 4 +; CHECK-NEXT: #dbg_assign(i1 undef, [[META13:![0-9]+]], !DIExpression(), [[DIASSIGNID16]], ptr [[A_ADDR]], !DIExpression(), [[META17:![0-9]+]]) +; CHECK-NEXT: store ptr [[A:%.*]], ptr [[A_ADDR]], align 8, !tbaa [[TBAA18:![0-9]+]] +; CHECK-NEXT: #dbg_declare(ptr [[A_ADDR]], [[META13]], !DIExpression(), [[META17]]) +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR2:[0-9]+]], !dbg [[DBG22:![0-9]+]] +; CHECK-NEXT: #dbg_declare(ptr [[I]], [[META14:![0-9]+]], !DIExpression(), [[META23:![0-9]+]]) +; CHECK-NEXT: store i32 0, ptr [[I]], align 4, !dbg [[META23]], !tbaa [[TBAA24:![0-9]+]] +; CHECK-NEXT: br label [[FOR_COND:%.*]], !dbg [[DBG22]] +; CHECK: for.cond: +; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[I]], align 4, !dbg [[DBG26:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 8, !dbg [[DBG28:![0-9]+]], !tbaa [[TBAA18]] +; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4, !dbg [[DBG29:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP0]], [[TMP2]], !dbg [[DBG30:![0-9]+]] +; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]], !dbg [[DBG31:![0-9]+]], !prof [[PROF32:![0-9]+]] +; CHECK: for.cond.cleanup: +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR2]], !dbg [[DBG33:![0-9]+]] +; CHECK-NEXT: br label [[FOR_END:%.*]] +; CHECK: for.body: +; CHECK-NEXT: [[TMP3:%.*]] = load ptr, ptr [[A_ADDR]], align 8, !dbg [[DBG34:![0-9]+]], !tbaa [[TBAA18]] +; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[I]], align 4, !dbg [[DBG35:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[IDXPROM:%.*]] = sext i32 [[TMP4]] to i64, !dbg [[DBG34]] +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 [[IDXPROM]], !dbg [[DBG34]] +; CHECK-NEXT: store i32 0, ptr [[ARRAYIDX]], align 4, !dbg [[DBG36:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: br label [[FOR_INC:%.*]], !dbg [[DBG34]] +; CHECK: for.inc: +; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[I]], align 4, !dbg [[DBG37:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP5]], 1, !dbg [[DBG37]] +; CHECK-NEXT: store i32 [[INC]], ptr [[I]], align 4, !dbg [[DBG37]], !tbaa [[TBAA24]] +; CHECK-NEXT: br label [[FOR_COND]], !dbg [[DBG33]], !llvm.loop [[LOOP38:![0-9]+]] +; CHECK: for.end: +; CHECK-NEXT: ret void, !dbg [[DBG40:![0-9]+]] +; +entry: + %A.addr = alloca ptr, align 8, !DIAssignID !16 + %i = alloca i32, align 4 + #dbg_assign(i1 undef, !13, !DIExpression(), !16, ptr %A.addr, !DIExpression(), !17) + store ptr %A, ptr %A.addr, align 8, !tbaa !18 + #dbg_declare(ptr %A.addr, !13, !DIExpression(), !17) + call void @llvm.lifetime.start.p0(i64 4, ptr %i) #2, !dbg !22 + #dbg_declare(ptr %i, !14, !DIExpression(), !23) + store i32 0, ptr %i, align 4, !dbg !23, !tbaa !24 + br label %for.cond, !dbg !22 + +for.cond: ; preds = %for.inc, %entry + %0 = load i32, ptr %i, align 4, !dbg !26, !tbaa !24 + %1 = load ptr, ptr %A.addr, align 8, !dbg !28, !tbaa !18 + %2 = load i32, ptr %1, align 4, !dbg !29, !tbaa !24 + %cmp = icmp slt i32 %0, %2, !dbg !30 + br i1 %cmp, label %for.body, label %for.cond.cleanup, !dbg !31, !prof !32 + +for.cond.cleanup: ; preds = %for.cond + call void @llvm.lifetime.end.p0(i64 4, ptr %i) #2, !dbg !33 + br label %for.end + +for.body: ; preds = %for.cond + %3 = load ptr, ptr %A.addr, align 8, !dbg !34, !tbaa !18 + %4 = load i32, ptr %i, align 4, !dbg !35, !tbaa !24 + %idxprom = sext i32 %4 to i64, !dbg !34 + %arrayidx = getelementptr inbounds i32, ptr %3, i64 %idxprom, !dbg !34 + store i32 0, ptr %arrayidx, align 4, !dbg !36, !tbaa !24 + br label %for.inc, !dbg !34 + +for.inc: ; preds = %for.body + %5 = load i32, ptr %i, align 4, !dbg !37, !tbaa !24 + %inc = add nsw i32 %5, 1, !dbg !37 + store i32 %inc, ptr %i, align 4, !dbg !37, !tbaa !24 + br label %for.cond, !dbg !33, !llvm.loop !38 + +for.end: ; preds = %for.cond.cleanup + ret void, !dbg !40 +} + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) +declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1 + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) +declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1 + +; Function Attrs: nounwind uwtable +define dso_local void @bar(ptr %A) #0 !dbg !41 { +; CHECK-LABEL: @bar( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[A_ADDR:%.*]] = alloca ptr, align 8 +; CHECK-NEXT: [[I:%.*]] = alloca i32, align 4 +; CHECK-NEXT: store ptr [[A:%.*]], ptr [[A_ADDR]], align 8, !tbaa [[TBAA18]] +; CHECK-NEXT: #dbg_declare(ptr [[A_ADDR]], [[META43:![0-9]+]], !DIExpression(), [[META46:![0-9]+]]) +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR2]], !dbg [[DBG47:![0-9]+]] +; CHECK-NEXT: #dbg_declare(ptr [[I]], [[META44:![0-9]+]], !DIExpression(), [[META48:![0-9]+]]) +; CHECK-NEXT: store i32 0, ptr [[I]], align 4, !dbg [[META48]], !tbaa [[TBAA24]] +; CHECK-NEXT: br label [[FOR_COND:%.*]], !dbg [[DBG47]] +; CHECK: for.cond: +; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[I]], align 4, !dbg [[DBG49:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 8, !dbg [[DBG51:![0-9]+]], !tbaa [[TBAA18]] +; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4, !dbg [[DBG52:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP0]], [[TMP2]], !dbg [[DBG53:![0-9]+]] +; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]], !dbg [[DBG54:![0-9]+]] +; CHECK: for.cond.cleanup: +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR2]], !dbg [[DBG55:![0-9]+]] +; CHECK-NEXT: br label [[FOR_END:%.*]] +; CHECK: for.body: +; CHECK-NEXT: [[TMP3:%.*]] = load ptr, ptr [[A_ADDR]], align 8, !dbg [[DBG56:![0-9]+]], !tbaa [[TBAA18]] +; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[I]], align 4, !dbg [[DBG57:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[IDXPROM:%.*]] = sext i32 [[TMP4]] to i64, !dbg [[DBG56]] +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 [[IDXPROM]], !dbg [[DBG56]] +; CHECK-NEXT: store i32 0, ptr [[ARRAYIDX]], align 4, !dbg [[DBG58:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: br label [[FOR_INC:%.*]], !dbg [[DBG56]] +; CHECK: for.inc: +; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[I]], align 4, !dbg [[DBG59:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP5]], 1, !dbg [[DBG59]] +; CHECK-NEXT: store i32 [[INC]], ptr [[I]], align 4, !dbg [[DBG59]], !tbaa [[TBAA24]] +; CHECK-NEXT: br label [[FOR_COND]], !dbg [[DBG55]], !llvm.loop [[LOOP60:![0-9]+]] +; CHECK: for.end: +; CHECK-NEXT: ret void, !dbg [[DBG62:![0-9]+]] +; +entry: + %A.addr = alloca ptr, align 8 + %i = alloca i32, align 4 + store ptr %A, ptr %A.addr, align 8, !tbaa !18 + #dbg_declare(ptr %A.addr, !43, !DIExpression(), !46) + call void @llvm.lifetime.start.p0(i64 4, ptr %i) #2, !dbg !47 + #dbg_declare(ptr %i, !44, !DIExpression(), !48) + store i32 0, ptr %i, align 4, !dbg !48, !tbaa !24 + br label %for.cond, !dbg !47 + +for.cond: ; preds = %for.inc, %entry + %0 = load i32, ptr %i, align 4, !dbg !49, !tbaa !24 + %1 = load ptr, ptr %A.addr, align 8, !dbg !51, !tbaa !18 + %2 = load i32, ptr %1, align 4, !dbg !52, !tbaa !24 + %cmp = icmp slt i32 %0, %2, !dbg !53 + br i1 %cmp, label %for.body, label %for.cond.cleanup, !dbg !54 + +for.cond.cleanup: ; preds = %for.cond + call void @llvm.lifetime.end.p0(i64 4, ptr %i) #2, !dbg !55 + br label %for.end + +for.body: ; preds = %for.cond + %3 = load ptr, ptr %A.addr, align 8, !dbg !56, !tbaa !18 + %4 = load i32, ptr %i, align 4, !dbg !57, !tbaa !24 + %idxprom = sext i32 %4 to i64, !dbg !56 + %arrayidx = getelementptr inbounds i32, ptr %3, i64 %idxprom, !dbg !56 + store i32 0, ptr %arrayidx, align 4, !dbg !58, !tbaa !24 + br label %for.inc, !dbg !56 + +for.inc: ; preds = %for.body + %5 = load i32, ptr %i, align 4, !dbg !59, !tbaa !24 + %inc = add nsw i32 %5, 1, !dbg !59 + store i32 %inc, ptr %i, align 4, !dbg !59, !tbaa !24 + br label %for.cond, !dbg !55, !llvm.loop !60 + +for.end: ; preds = %for.cond.cleanup + ret void, !dbg !62 +} + +attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #1 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) } +attributes #2 = { nounwind } + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!3, !4, !5} +!llvm.ident = !{!6} + +!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 11.0.0 (git@github.com:llvm/llvm-project.git 1d5da8cd30fce1c0a2c2fa6ba656dbfaa36192c8)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, splitDebugInlining: false, nameTableKind: None) +!1 = !DIFile(filename: "various_ir_values.c", directory: "/data/build/llvm-project") +!2 = !{} +!3 = !{i32 7, !"Dwarf Version", i32 4} +!4 = !{i32 2, !"Debug Info Version", i32 3} +!5 = !{i32 1, !"wchar_size", i32 4} +!6 = !{!"clang version 11.0.0 (git@github.com:llvm/llvm-project.git 1d5da8cd30fce1c0a2c2fa6ba656dbfaa36192c8)"} +!7 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !8, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !12) +!8 = !DISubroutineType(types: !9) +!9 = !{null, !10} +!10 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !11, size: 64) +!11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!12 = !{!13, !14} +!13 = !DILocalVariable(name: "A", arg: 1, scope: !7, file: !1, line: 1, type: !10) +!14 = !DILocalVariable(name: "i", scope: !15, file: !1, line: 3, type: !11) +!15 = distinct !DILexicalBlock(scope: !7, file: !1, line: 3, column: 3) +!16 = distinct !DIAssignID() +!17 = !DILocation(line: 1, column: 15, scope: !7) +!18 = !{!19, !19, i64 0} +!19 = !{!"any pointer", !20, i64 0} +!20 = !{!"omnipotent char", !21, i64 0} +!21 = !{!"Simple C/C++ TBAA"} +!22 = !DILocation(line: 3, column: 8, scope: !15) +!23 = !DILocation(line: 3, column: 12, scope: !15) +!24 = !{!25, !25, i64 0} +!25 = !{!"int", !20, i64 0} +!26 = !DILocation(line: 3, column: 19, scope: !27) +!27 = distinct !DILexicalBlock(scope: !15, file: !1, line: 3, column: 3) +!28 = !DILocation(line: 3, column: 24, scope: !27) +!29 = !DILocation(line: 3, column: 23, scope: !27) +!30 = !DILocation(line: 3, column: 21, scope: !27) +!31 = !DILocation(line: 3, column: 3, scope: !15) +!32 = !{!"branch_weights", i32 1, i32 1048575} +!33 = !DILocation(line: 3, column: 3, scope: !27) +!34 = !DILocation(line: 4, column: 5, scope: !27) +!35 = !DILocation(line: 4, column: 7, scope: !27) +!36 = !DILocation(line: 4, column: 10, scope: !27) +!37 = !DILocation(line: 3, column: 27, scope: !27) +!38 = distinct !{!38, !31, !39} +!39 = !DILocation(line: 4, column: 12, scope: !15) +!40 = !DILocation(line: 5, column: 1, scope: !7) +!41 = distinct !DISubprogram(name: "bar", scope: !1, file: !1, line: 7, type: !8, scopeLine: 7, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !42) +!42 = !{!43, !44} +!43 = !DILocalVariable(name: "A", arg: 1, scope: !41, file: !1, line: 7, type: !10) +!44 = !DILocalVariable(name: "i", scope: !45, file: !1, line: 9, type: !11) +!45 = distinct !DILexicalBlock(scope: !41, file: !1, line: 9, column: 3) +!46 = !DILocation(line: 7, column: 15, scope: !41) +!47 = !DILocation(line: 9, column: 8, scope: !45) +!48 = !DILocation(line: 9, column: 12, scope: !45) +!49 = !DILocation(line: 9, column: 19, scope: !50) +!50 = distinct !DILexicalBlock(scope: !45, file: !1, line: 9, column: 3) +!51 = !DILocation(line: 9, column: 24, scope: !50) +!52 = !DILocation(line: 9, column: 23, scope: !50) +!53 = !DILocation(line: 9, column: 21, scope: !50) +!54 = !DILocation(line: 9, column: 3, scope: !45) +!55 = !DILocation(line: 9, column: 3, scope: !50) +!56 = !DILocation(line: 10, column: 5, scope: !50) +!57 = !DILocation(line: 10, column: 7, scope: !50) +!58 = !DILocation(line: 10, column: 10, scope: !50) +!59 = !DILocation(line: 9, column: 27, scope: !50) +!60 = distinct !{!60, !54, !61} +!61 = !DILocation(line: 10, column: 12, scope: !45) +!62 = !DILocation(line: 11, column: 1, scope: !41) diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.funcsig.transitiveglobals.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.funcsig.transitiveglobals.expected new file mode 100644 index 000000000000..e2c426029a6b --- /dev/null +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.funcsig.transitiveglobals.expected @@ -0,0 +1,299 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals smart +; Just run it through opt, no passes needed. +; RUN: opt < %s -S --write-experimental-debuginfo=true | FileCheck %s + +; ModuleID = 'various_ir_values.c' +source_filename = "various_ir_values.c" +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +; Function Attrs: nounwind uwtable +define dso_local void @foo(ptr %A) #0 !dbg !7 { +; CHECK-LABEL: @foo( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[A_ADDR:%.*]] = alloca ptr, align 8, !DIAssignID [[DIASSIGNID16:![0-9]+]] +; CHECK-NEXT: [[I:%.*]] = alloca i32, align 4 +; CHECK-NEXT: #dbg_assign(i1 undef, [[META13:![0-9]+]], !DIExpression(), [[DIASSIGNID16]], ptr [[A_ADDR]], !DIExpression(), [[META17:![0-9]+]]) +; CHECK-NEXT: store ptr [[A:%.*]], ptr [[A_ADDR]], align 8, !tbaa [[TBAA18:![0-9]+]] +; CHECK-NEXT: #dbg_declare(ptr [[A_ADDR]], [[META13]], !DIExpression(), [[META17]]) +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR2:[0-9]+]], !dbg [[DBG22:![0-9]+]] +; CHECK-NEXT: #dbg_declare(ptr [[I]], [[META14:![0-9]+]], !DIExpression(), [[META23:![0-9]+]]) +; CHECK-NEXT: store i32 0, ptr [[I]], align 4, !dbg [[META23]], !tbaa [[TBAA24:![0-9]+]] +; CHECK-NEXT: br label [[FOR_COND:%.*]], !dbg [[DBG22]] +; CHECK: for.cond: +; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[I]], align 4, !dbg [[DBG26:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 8, !dbg [[DBG28:![0-9]+]], !tbaa [[TBAA18]] +; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4, !dbg [[DBG29:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP0]], [[TMP2]], !dbg [[DBG30:![0-9]+]] +; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]], !dbg [[DBG31:![0-9]+]], !prof [[PROF32:![0-9]+]] +; CHECK: for.cond.cleanup: +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR2]], !dbg [[DBG33:![0-9]+]] +; CHECK-NEXT: br label [[FOR_END:%.*]] +; CHECK: for.body: +; CHECK-NEXT: [[TMP3:%.*]] = load ptr, ptr [[A_ADDR]], align 8, !dbg [[DBG34:![0-9]+]], !tbaa [[TBAA18]] +; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[I]], align 4, !dbg [[DBG35:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[IDXPROM:%.*]] = sext i32 [[TMP4]] to i64, !dbg [[DBG34]] +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 [[IDXPROM]], !dbg [[DBG34]] +; CHECK-NEXT: store i32 0, ptr [[ARRAYIDX]], align 4, !dbg [[DBG36:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: br label [[FOR_INC:%.*]], !dbg [[DBG34]] +; CHECK: for.inc: +; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[I]], align 4, !dbg [[DBG37:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP5]], 1, !dbg [[DBG37]] +; CHECK-NEXT: store i32 [[INC]], ptr [[I]], align 4, !dbg [[DBG37]], !tbaa [[TBAA24]] +; CHECK-NEXT: br label [[FOR_COND]], !dbg [[DBG33]], !llvm.loop [[LOOP38:![0-9]+]] +; CHECK: for.end: +; CHECK-NEXT: ret void, !dbg [[DBG40:![0-9]+]] +; +entry: + %A.addr = alloca ptr, align 8, !DIAssignID !16 + %i = alloca i32, align 4 + #dbg_assign(i1 undef, !13, !DIExpression(), !16, ptr %A.addr, !DIExpression(), !17) + store ptr %A, ptr %A.addr, align 8, !tbaa !18 + #dbg_declare(ptr %A.addr, !13, !DIExpression(), !17) + call void @llvm.lifetime.start.p0(i64 4, ptr %i) #2, !dbg !22 + #dbg_declare(ptr %i, !14, !DIExpression(), !23) + store i32 0, ptr %i, align 4, !dbg !23, !tbaa !24 + br label %for.cond, !dbg !22 + +for.cond: ; preds = %for.inc, %entry + %0 = load i32, ptr %i, align 4, !dbg !26, !tbaa !24 + %1 = load ptr, ptr %A.addr, align 8, !dbg !28, !tbaa !18 + %2 = load i32, ptr %1, align 4, !dbg !29, !tbaa !24 + %cmp = icmp slt i32 %0, %2, !dbg !30 + br i1 %cmp, label %for.body, label %for.cond.cleanup, !dbg !31, !prof !32 + +for.cond.cleanup: ; preds = %for.cond + call void @llvm.lifetime.end.p0(i64 4, ptr %i) #2, !dbg !33 + br label %for.end + +for.body: ; preds = %for.cond + %3 = load ptr, ptr %A.addr, align 8, !dbg !34, !tbaa !18 + %4 = load i32, ptr %i, align 4, !dbg !35, !tbaa !24 + %idxprom = sext i32 %4 to i64, !dbg !34 + %arrayidx = getelementptr inbounds i32, ptr %3, i64 %idxprom, !dbg !34 + store i32 0, ptr %arrayidx, align 4, !dbg !36, !tbaa !24 + br label %for.inc, !dbg !34 + +for.inc: ; preds = %for.body + %5 = load i32, ptr %i, align 4, !dbg !37, !tbaa !24 + %inc = add nsw i32 %5, 1, !dbg !37 + store i32 %inc, ptr %i, align 4, !dbg !37, !tbaa !24 + br label %for.cond, !dbg !33, !llvm.loop !38 + +for.end: ; preds = %for.cond.cleanup + ret void, !dbg !40 +} + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) +declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1 + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) +declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1 + +; Function Attrs: nounwind uwtable +define dso_local void @bar(ptr %A) #0 !dbg !41 { +; CHECK-LABEL: @bar( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[A_ADDR:%.*]] = alloca ptr, align 8 +; CHECK-NEXT: [[I:%.*]] = alloca i32, align 4 +; CHECK-NEXT: store ptr [[A:%.*]], ptr [[A_ADDR]], align 8, !tbaa [[TBAA18]] +; CHECK-NEXT: #dbg_declare(ptr [[A_ADDR]], [[META43:![0-9]+]], !DIExpression(), [[META46:![0-9]+]]) +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR2]], !dbg [[DBG47:![0-9]+]] +; CHECK-NEXT: #dbg_declare(ptr [[I]], [[META44:![0-9]+]], !DIExpression(), [[META48:![0-9]+]]) +; CHECK-NEXT: store i32 0, ptr [[I]], align 4, !dbg [[META48]], !tbaa [[TBAA24]] +; CHECK-NEXT: br label [[FOR_COND:%.*]], !dbg [[DBG47]] +; CHECK: for.cond: +; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[I]], align 4, !dbg [[DBG49:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 8, !dbg [[DBG51:![0-9]+]], !tbaa [[TBAA18]] +; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4, !dbg [[DBG52:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP0]], [[TMP2]], !dbg [[DBG53:![0-9]+]] +; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]], !dbg [[DBG54:![0-9]+]] +; CHECK: for.cond.cleanup: +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR2]], !dbg [[DBG55:![0-9]+]] +; CHECK-NEXT: br label [[FOR_END:%.*]] +; CHECK: for.body: +; CHECK-NEXT: [[TMP3:%.*]] = load ptr, ptr [[A_ADDR]], align 8, !dbg [[DBG56:![0-9]+]], !tbaa [[TBAA18]] +; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[I]], align 4, !dbg [[DBG57:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[IDXPROM:%.*]] = sext i32 [[TMP4]] to i64, !dbg [[DBG56]] +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 [[IDXPROM]], !dbg [[DBG56]] +; CHECK-NEXT: store i32 0, ptr [[ARRAYIDX]], align 4, !dbg [[DBG58:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: br label [[FOR_INC:%.*]], !dbg [[DBG56]] +; CHECK: for.inc: +; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[I]], align 4, !dbg [[DBG59:![0-9]+]], !tbaa [[TBAA24]] +; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP5]], 1, !dbg [[DBG59]] +; CHECK-NEXT: store i32 [[INC]], ptr [[I]], align 4, !dbg [[DBG59]], !tbaa [[TBAA24]] +; CHECK-NEXT: br label [[FOR_COND]], !dbg [[DBG55]], !llvm.loop [[LOOP60:![0-9]+]] +; CHECK: for.end: +; CHECK-NEXT: ret void, !dbg [[DBG62:![0-9]+]] +; +entry: + %A.addr = alloca ptr, align 8 + %i = alloca i32, align 4 + store ptr %A, ptr %A.addr, align 8, !tbaa !18 + #dbg_declare(ptr %A.addr, !43, !DIExpression(), !46) + call void @llvm.lifetime.start.p0(i64 4, ptr %i) #2, !dbg !47 + #dbg_declare(ptr %i, !44, !DIExpression(), !48) + store i32 0, ptr %i, align 4, !dbg !48, !tbaa !24 + br label %for.cond, !dbg !47 + +for.cond: ; preds = %for.inc, %entry + %0 = load i32, ptr %i, align 4, !dbg !49, !tbaa !24 + %1 = load ptr, ptr %A.addr, align 8, !dbg !51, !tbaa !18 + %2 = load i32, ptr %1, align 4, !dbg !52, !tbaa !24 + %cmp = icmp slt i32 %0, %2, !dbg !53 + br i1 %cmp, label %for.body, label %for.cond.cleanup, !dbg !54 + +for.cond.cleanup: ; preds = %for.cond + call void @llvm.lifetime.end.p0(i64 4, ptr %i) #2, !dbg !55 + br label %for.end + +for.body: ; preds = %for.cond + %3 = load ptr, ptr %A.addr, align 8, !dbg !56, !tbaa !18 + %4 = load i32, ptr %i, align 4, !dbg !57, !tbaa !24 + %idxprom = sext i32 %4 to i64, !dbg !56 + %arrayidx = getelementptr inbounds i32, ptr %3, i64 %idxprom, !dbg !56 + store i32 0, ptr %arrayidx, align 4, !dbg !58, !tbaa !24 + br label %for.inc, !dbg !56 + +for.inc: ; preds = %for.body + %5 = load i32, ptr %i, align 4, !dbg !59, !tbaa !24 + %inc = add nsw i32 %5, 1, !dbg !59 + store i32 %inc, ptr %i, align 4, !dbg !59, !tbaa !24 + br label %for.cond, !dbg !55, !llvm.loop !60 + +for.end: ; preds = %for.cond.cleanup + ret void, !dbg !62 +} + +attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #1 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) } +attributes #2 = { nounwind } + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!3, !4, !5} +!llvm.ident = !{!6} + +!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 11.0.0 (git@github.com:llvm/llvm-project.git 1d5da8cd30fce1c0a2c2fa6ba656dbfaa36192c8)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, splitDebugInlining: false, nameTableKind: None) +!1 = !DIFile(filename: "various_ir_values.c", directory: "/data/build/llvm-project") +!2 = !{} +!3 = !{i32 7, !"Dwarf Version", i32 4} +!4 = !{i32 2, !"Debug Info Version", i32 3} +!5 = !{i32 1, !"wchar_size", i32 4} +!6 = !{!"clang version 11.0.0 (git@github.com:llvm/llvm-project.git 1d5da8cd30fce1c0a2c2fa6ba656dbfaa36192c8)"} +!7 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !8, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !12) +!8 = !DISubroutineType(types: !9) +!9 = !{null, !10} +!10 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !11, size: 64) +!11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!12 = !{!13, !14} +!13 = !DILocalVariable(name: "A", arg: 1, scope: !7, file: !1, line: 1, type: !10) +!14 = !DILocalVariable(name: "i", scope: !15, file: !1, line: 3, type: !11) +!15 = distinct !DILexicalBlock(scope: !7, file: !1, line: 3, column: 3) +!16 = distinct !DIAssignID() +!17 = !DILocation(line: 1, column: 15, scope: !7) +!18 = !{!19, !19, i64 0} +!19 = !{!"any pointer", !20, i64 0} +!20 = !{!"omnipotent char", !21, i64 0} +!21 = !{!"Simple C/C++ TBAA"} +!22 = !DILocation(line: 3, column: 8, scope: !15) +!23 = !DILocation(line: 3, column: 12, scope: !15) +!24 = !{!25, !25, i64 0} +!25 = !{!"int", !20, i64 0} +!26 = !DILocation(line: 3, column: 19, scope: !27) +!27 = distinct !DILexicalBlock(scope: !15, file: !1, line: 3, column: 3) +!28 = !DILocation(line: 3, column: 24, scope: !27) +!29 = !DILocation(line: 3, column: 23, scope: !27) +!30 = !DILocation(line: 3, column: 21, scope: !27) +!31 = !DILocation(line: 3, column: 3, scope: !15) +!32 = !{!"branch_weights", i32 1, i32 1048575} +!33 = !DILocation(line: 3, column: 3, scope: !27) +!34 = !DILocation(line: 4, column: 5, scope: !27) +!35 = !DILocation(line: 4, column: 7, scope: !27) +!36 = !DILocation(line: 4, column: 10, scope: !27) +!37 = !DILocation(line: 3, column: 27, scope: !27) +!38 = distinct !{!38, !31, !39} +!39 = !DILocation(line: 4, column: 12, scope: !15) +!40 = !DILocation(line: 5, column: 1, scope: !7) +!41 = distinct !DISubprogram(name: "bar", scope: !1, file: !1, line: 7, type: !8, scopeLine: 7, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !42) +!42 = !{!43, !44} +!43 = !DILocalVariable(name: "A", arg: 1, scope: !41, file: !1, line: 7, type: !10) +!44 = !DILocalVariable(name: "i", scope: !45, file: !1, line: 9, type: !11) +!45 = distinct !DILexicalBlock(scope: !41, file: !1, line: 9, column: 3) +!46 = !DILocation(line: 7, column: 15, scope: !41) +!47 = !DILocation(line: 9, column: 8, scope: !45) +!48 = !DILocation(line: 9, column: 12, scope: !45) +!49 = !DILocation(line: 9, column: 19, scope: !50) +!50 = distinct !DILexicalBlock(scope: !45, file: !1, line: 9, column: 3) +!51 = !DILocation(line: 9, column: 24, scope: !50) +!52 = !DILocation(line: 9, column: 23, scope: !50) +!53 = !DILocation(line: 9, column: 21, scope: !50) +!54 = !DILocation(line: 9, column: 3, scope: !45) +!55 = !DILocation(line: 9, column: 3, scope: !50) +!56 = !DILocation(line: 10, column: 5, scope: !50) +!57 = !DILocation(line: 10, column: 7, scope: !50) +!58 = !DILocation(line: 10, column: 10, scope: !50) +!59 = !DILocation(line: 9, column: 27, scope: !50) +!60 = distinct !{!60, !54, !61} +!61 = !DILocation(line: 10, column: 12, scope: !45) +!62 = !DILocation(line: 11, column: 1, scope: !41) +;. +; CHECK: [[META0:![0-9]+]] = distinct !DICompileUnit(language: DW_LANG_C99, file: [[META1:![0-9]+]], producer: "{{.*}}clang version {{.*}}", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: [[META2:![0-9]+]], splitDebugInlining: false, nameTableKind: None) +; CHECK: [[META1]] = !DIFile(filename: "various_ir_values.c", directory: {{.*}}) +; CHECK: [[META2]] = !{} +; CHECK: [[META7:![0-9]+]] = distinct !DISubprogram(name: "foo", scope: [[META1]], file: [[META1]], line: 1, type: [[META8:![0-9]+]], scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META12:![0-9]+]]) +; CHECK: [[META8]] = !DISubroutineType(types: [[META9:![0-9]+]]) +; CHECK: [[META9]] = !{null, [[META10:![0-9]+]]} +; CHECK: [[META10]] = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: [[META11:![0-9]+]], size: 64) +; CHECK: [[META11]] = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +; CHECK: [[META12]] = !{[[META13]], [[META14]]} +; CHECK: [[META13]] = !DILocalVariable(name: "A", arg: 1, scope: [[META7]], file: [[META1]], line: 1, type: [[META10]]) +; CHECK: [[META14]] = !DILocalVariable(name: "i", scope: [[META15:![0-9]+]], file: [[META1]], line: 3, type: [[META11]]) +; CHECK: [[META15]] = distinct !DILexicalBlock(scope: [[META7]], file: [[META1]], line: 3, column: 3) +; CHECK: [[DIASSIGNID16]] = distinct !DIAssignID() +; CHECK: [[META17]] = !DILocation(line: 1, column: 15, scope: [[META7]]) +; CHECK: [[TBAA18]] = !{[[META19:![0-9]+]], [[META19]], i64 0} +; CHECK: [[META19]] = !{!"any pointer", [[META20:![0-9]+]], i64 0} +; CHECK: [[META20]] = !{!"omnipotent char", [[META21:![0-9]+]], i64 0} +; CHECK: [[META21]] = !{!"Simple C/C++ TBAA"} +; CHECK: [[DBG22]] = !DILocation(line: 3, column: 8, scope: [[META15]]) +; CHECK: [[META23]] = !DILocation(line: 3, column: 12, scope: [[META15]]) +; CHECK: [[TBAA24]] = !{[[META25:![0-9]+]], [[META25]], i64 0} +; CHECK: [[META25]] = !{!"int", [[META20]], i64 0} +; CHECK: [[DBG26]] = !DILocation(line: 3, column: 19, scope: [[META27:![0-9]+]]) +; CHECK: [[META27]] = distinct !DILexicalBlock(scope: [[META15]], file: [[META1]], line: 3, column: 3) +; CHECK: [[DBG28]] = !DILocation(line: 3, column: 24, scope: [[META27]]) +; CHECK: [[DBG29]] = !DILocation(line: 3, column: 23, scope: [[META27]]) +; CHECK: [[DBG30]] = !DILocation(line: 3, column: 21, scope: [[META27]]) +; CHECK: [[DBG31]] = !DILocation(line: 3, column: 3, scope: [[META15]]) +; CHECK: [[PROF32]] = !{!"branch_weights", i32 1, i32 1048575} +; CHECK: [[DBG33]] = !DILocation(line: 3, column: 3, scope: [[META27]]) +; CHECK: [[DBG34]] = !DILocation(line: 4, column: 5, scope: [[META27]]) +; CHECK: [[DBG35]] = !DILocation(line: 4, column: 7, scope: [[META27]]) +; CHECK: [[DBG36]] = !DILocation(line: 4, column: 10, scope: [[META27]]) +; CHECK: [[DBG37]] = !DILocation(line: 3, column: 27, scope: [[META27]]) +; CHECK: [[LOOP38]] = distinct !{[[LOOP38]], [[DBG31]], [[META39:![0-9]+]]} +; CHECK: [[META39]] = !DILocation(line: 4, column: 12, scope: [[META15]]) +; CHECK: [[DBG40]] = !DILocation(line: 5, column: 1, scope: [[META7]]) +; CHECK: [[META41:![0-9]+]] = distinct !DISubprogram(name: "bar", scope: [[META1]], file: [[META1]], line: 7, type: [[META8]], scopeLine: 7, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META42:![0-9]+]]) +; CHECK: [[META42]] = !{[[META43]], [[META44]]} +; CHECK: [[META43]] = !DILocalVariable(name: "A", arg: 1, scope: [[META41]], file: [[META1]], line: 7, type: [[META10]]) +; CHECK: [[META44]] = !DILocalVariable(name: "i", scope: [[META45:![0-9]+]], file: [[META1]], line: 9, type: [[META11]]) +; CHECK: [[META45]] = distinct !DILexicalBlock(scope: [[META41]], file: [[META1]], line: 9, column: 3) +; CHECK: [[META46]] = !DILocation(line: 7, column: 15, scope: [[META41]]) +; CHECK: [[DBG47]] = !DILocation(line: 9, column: 8, scope: [[META45]]) +; CHECK: [[META48]] = !DILocation(line: 9, column: 12, scope: [[META45]]) +; CHECK: [[DBG49]] = !DILocation(line: 9, column: 19, scope: [[META50:![0-9]+]]) +; CHECK: [[META50]] = distinct !DILexicalBlock(scope: [[META45]], file: [[META1]], line: 9, column: 3) +; CHECK: [[DBG51]] = !DILocation(line: 9, column: 24, scope: [[META50]]) +; CHECK: [[DBG52]] = !DILocation(line: 9, column: 23, scope: [[META50]]) +; CHECK: [[DBG53]] = !DILocation(line: 9, column: 21, scope: [[META50]]) +; CHECK: [[DBG54]] = !DILocation(line: 9, column: 3, scope: [[META45]]) +; CHECK: [[DBG55]] = !DILocation(line: 9, column: 3, scope: [[META50]]) +; CHECK: [[DBG56]] = !DILocation(line: 10, column: 5, scope: [[META50]]) +; CHECK: [[DBG57]] = !DILocation(line: 10, column: 7, scope: [[META50]]) +; CHECK: [[DBG58]] = !DILocation(line: 10, column: 10, scope: [[META50]]) +; CHECK: [[DBG59]] = !DILocation(line: 9, column: 27, scope: [[META50]]) +; CHECK: [[LOOP60]] = distinct !{[[LOOP60]], [[DBG54]], [[META61:![0-9]+]]} +; CHECK: [[META61]] = !DILocation(line: 10, column: 12, scope: [[META45]]) +; CHECK: [[DBG62]] = !DILocation(line: 11, column: 1, scope: [[META41]]) +;. diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/various_ir_values_dbgrecords.test b/llvm/test/tools/UpdateTestChecks/update_test_checks/various_ir_values_dbgrecords.test new file mode 100644 index 000000000000..9cc77d894d62 --- /dev/null +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/various_ir_values_dbgrecords.test @@ -0,0 +1,24 @@ +## Basic test checking that update_test_checks.py works correctly on various "IR value" kinds +# RUN: cp -f %S/Inputs/various_ir_values_dbgrecords.ll %t.ll && %update_test_checks %t.ll +# RUN: diff -u %t.ll %S/Inputs/various_ir_values_dbgrecords.ll.expected +## Check that running the script again does not change the result: +# RUN: %update_test_checks %t.ll +# RUN: diff -u %t.ll %S/Inputs/various_ir_values_dbgrecords.ll.expected +## Also try the --function-signature flag +# RUN: %update_test_checks %t.ll --function-signature +# RUN: diff -u %t.ll %S/Inputs/various_ir_values_dbgrecords.ll.funcsig.expected +## Verify that running without the --function-signature flag does not removes +## the -SAME: lines since the generated file will have --function-signature in +## an UTC_ARGS: comment in the first line (from the invocation above) which is +## added to the update invocation below. +# RUN: %update_test_checks %t.ll +# RUN: diff -u %t.ll %S/Inputs/various_ir_values_dbgrecords.ll.funcsig.expected +## Also try the --check-globals flag +# RUN: %update_test_checks %t.ll --check-globals +# RUN: diff -u %t.ll %S/Inputs/various_ir_values_dbgrecords.ll.funcsig.globals.expected +# RUN: cp -f %S/Inputs/various_ir_values_dbgrecords.ll %t.ll && %update_test_checks %t.ll --function-signature --check-globals all +# RUN: diff -u %t.ll %S/Inputs/various_ir_values_dbgrecords.ll.funcsig.globals.expected +# RUN: cp -f %S/Inputs/various_ir_values_dbgrecords.ll %t.ll && %update_test_checks %t.ll --check-globals none +# RUN: diff -u %t.ll %S/Inputs/various_ir_values_dbgrecords.ll.funcsig.noglobals.expected +# RUN: cp -f %S/Inputs/various_ir_values_dbgrecords.ll %t.ll && %update_test_checks %t.ll --check-globals smart +# RUN: diff -u %t.ll %S/Inputs/various_ir_values_dbgrecords.ll.funcsig.transitiveglobals.expected diff --git a/llvm/utils/UpdateTestChecks/common.py b/llvm/utils/UpdateTestChecks/common.py index 15d3d5e527d6..5595e6f41755 100644 --- a/llvm/utils/UpdateTestChecks/common.py +++ b/llvm/utils/UpdateTestChecks/common.py @@ -1082,10 +1082,10 @@ ir_nameless_values = [ NamelessValue(r"TBAA_STRUCT", "!", r"!tbaa.struct ", r"![0-9]+", None), NamelessValue(r"RNG", "!", r"!range ", r"![0-9]+", None), NamelessValue(r"LOOP", "!", r"!llvm.loop ", r"![0-9]+", None), - NamelessValue(r"META", "!", r"metadata ", r"![0-9]+", None), NamelessValue(r"META", "!", r"", r"![0-9]+", r"(?:distinct |)!.*"), NamelessValue(r"ACC_GRP", "!", r"!llvm.access.group ", r"![0-9]+", None), NamelessValue(r"META", "!", r"![a-z.]+ ", r"![0-9]+", None), + NamelessValue(r"META", "!", r"[, (]", r"![0-9]+", None), ] global_nameless_values = [ -- GitLab From 792d437b56adfb3416daf8105942d4899fb82763 Mon Sep 17 00:00:00 2001 From: NagyDonat Date: Wed, 17 Apr 2024 12:52:23 +0200 Subject: [PATCH 034/862] [clang-tidy NFC] Fix a typo in docs for sizeof-expression (#88912) "Till heaven and earth pass, one jot, or one tittle shall not pass of the law" --- .../docs/clang-tidy/checks/bugprone/sizeof-expression.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang-tools-extra/docs/clang-tidy/checks/bugprone/sizeof-expression.rst b/clang-tools-extra/docs/clang-tidy/checks/bugprone/sizeof-expression.rst index a3e88b837d37..c37df1706eb4 100644 --- a/clang-tools-extra/docs/clang-tidy/checks/bugprone/sizeof-expression.rst +++ b/clang-tools-extra/docs/clang-tidy/checks/bugprone/sizeof-expression.rst @@ -190,6 +190,6 @@ Options .. option:: WarnOnSizeOfPointerToAggregate - When `true, the check will warn on an expression like + When `true`, the check will warn on an expression like ``sizeof(expr)`` where the expression is a pointer to aggregate. Default is `true`. -- GitLab From 5f3e106de3cd5ce6d7ba37fb11f6ad740cb430c5 Mon Sep 17 00:00:00 2001 From: Pavel Labath Date: Wed, 17 Apr 2024 13:05:15 +0200 Subject: [PATCH 035/862] [lldb/linux] Make sure the process continues running after a detach (#88494) Fixes #85084 Whenever an inferior thread stops, lldb-server sends a SIGSTOP to all other threads in the process to force them to stop as well. If those threads stop on their own before they get a signal, this SIGSTOP will remain pending and be delivered the next time the process resumes. Normally, this is not a problem, because lldb-server will detect this stale SIGSTOP and resume the process. However, if we detach from the process while it has these SIGSTOPs pending, they will get immediately delivered, and the process will remain stopped (most likely forever). This patch fixes that by sending a SIGCONT just before detaching from the process. This signal cancels out any pending SIGSTOPs, and ensures it is able to run after we detach. It does have one somewhat unfortunate side-effect that in that the process's SIGCONT handler (if it has one) will get executed spuriously (from the process's POV). This could be _sometimes_ avoided by tracking which threads got send a SIGSTOP, and whether those threads stopped due to it. From what I could tell by observing its behavior, this is what gdb does. I have not tried to replicate that behavior here because it adds a nontrivial amount of complexity and the result is still uncertain -- we still need to send a SIGCONT (and execute the handler) when any thread stops for some other reason (and leaves our SIGSTOP hanging). Furthermore, since SIGSTOPs don't stack, it's also possible that our SIGSTOP/SIGCONT combination will cancel a genuine SIGSTOP being sent to the debugger application (by someone else), and there is nothing we can do about that. For this reason I think it's simplest and most predictible to just always send a SIGCONT when detaching, but if it turns out this is breaking something, we can consider implementing something more elaborate. One alternative I did try is to use PTRACE_INTERRUPT to suspend the threads instead of a SIGSTOP. PTRACE_INTERUPT requires using PTRACE_SEIZE to attach to the process, which also made this solution somewhat complicated, but the main problem with that approach is that PTRACE_INTERRUPT is not considered to be a signal-delivery-stop, which means it's not possible to resume it while injecting another signal to the inferior (which some of our tests expect to be able to do). This limitation could be worked around by forcing the thread into a signal delivery stop whenever we need to do this, but this additional complication is what made me think this approach is also not worthwhile. This patch should fix (at least some of) the problems with TestConcurrentVFork, but I've also added a dedicated test for checking that a process keeps running after we detach. Although the problem I'm fixing here is linux-specific, the core functinoality of not stopping after a detach should function the same way everywhere. --- .../Process/Linux/NativeProcessLinux.cpp | 4 ++ .../commands/process/detach-resumes/Makefile | 4 ++ .../detach-resumes/TestDetachResumes.py | 59 +++++++++++++++++++ .../commands/process/detach-resumes/main.cpp | 48 +++++++++++++++ .../concurrent_vfork/TestConcurrentVFork.py | 16 ----- 5 files changed, 115 insertions(+), 16 deletions(-) create mode 100644 lldb/test/API/commands/process/detach-resumes/Makefile create mode 100644 lldb/test/API/commands/process/detach-resumes/TestDetachResumes.py create mode 100644 lldb/test/API/commands/process/detach-resumes/main.cpp diff --git a/lldb/source/Plugins/Process/Linux/NativeProcessLinux.cpp b/lldb/source/Plugins/Process/Linux/NativeProcessLinux.cpp index 5d2b4b03fe60..59fc8726b767 100644 --- a/lldb/source/Plugins/Process/Linux/NativeProcessLinux.cpp +++ b/lldb/source/Plugins/Process/Linux/NativeProcessLinux.cpp @@ -1089,6 +1089,10 @@ Status NativeProcessLinux::Detach() { if (GetID() == LLDB_INVALID_PROCESS_ID) return error; + // Cancel out any SIGSTOPs we may have sent while stopping the process. + // Otherwise, the process may stop as soon as we detach from it. + kill(GetID(), SIGCONT); + for (const auto &thread : m_threads) { Status e = Detach(thread->GetID()); if (e.Fail()) diff --git a/lldb/test/API/commands/process/detach-resumes/Makefile b/lldb/test/API/commands/process/detach-resumes/Makefile new file mode 100644 index 000000000000..c46619c66234 --- /dev/null +++ b/lldb/test/API/commands/process/detach-resumes/Makefile @@ -0,0 +1,4 @@ +CXX_SOURCES := main.cpp +ENABLE_THREADS := YES + +include Makefile.rules diff --git a/lldb/test/API/commands/process/detach-resumes/TestDetachResumes.py b/lldb/test/API/commands/process/detach-resumes/TestDetachResumes.py new file mode 100644 index 000000000000..57727294ddc3 --- /dev/null +++ b/lldb/test/API/commands/process/detach-resumes/TestDetachResumes.py @@ -0,0 +1,59 @@ +""" +Test that the process continues running after we detach from it. +""" + +import lldb +import time +from lldbsuite.test.decorators import * +from lldbsuite.test.lldbtest import * +from lldbsuite.test import lldbutil + + +class DetachResumesTestCase(TestBase): + NO_DEBUG_INFO_TESTCASE = True + + def test_detach_resumes(self): + self.build() + exe = self.getBuildArtifact() + + # The inferior will use this file to let us know it is ready to be + # attached. + sync_file_path = lldbutil.append_to_process_working_directory( + self, "sync_file_%d" % (int(time.time())) + ) + + # And this one to let us know it is running after we've detached from + # it. + exit_file_path = lldbutil.append_to_process_working_directory( + self, "exit_file_%d" % (int(time.time())) + ) + + popen = self.spawnSubprocess( + self.getBuildArtifact(exe), [sync_file_path, exit_file_path] + ) + lldbutil.wait_for_file_on_target(self, sync_file_path) + + self.runCmd("process attach -p " + str(popen.pid)) + + # Set a breakpoint at a place that will be called by multiple threads + # simultaneously. On systems (e.g. linux) where the debugger needs to + # send signals to suspend threads, these signals will race with threads + # hitting the breakpoint (and stopping on their own). + bpno = lldbutil.run_break_set_by_symbol(self, "break_here") + + # And let the inferior know it can call the function. + self.runCmd("expr -- wait_for_debugger_flag = false") + + self.runCmd("continue") + + self.expect( + "thread list", + STOPPED_DUE_TO_BREAKPOINT, + substrs=["stopped", "stop reason = breakpoint"], + ) + + # Detach, the process should keep running after this, and not be stopped + # by the signals that the debugger may have used to suspend the threads. + self.runCmd("detach") + + lldbutil.wait_for_file_on_target(self, exit_file_path) diff --git a/lldb/test/API/commands/process/detach-resumes/main.cpp b/lldb/test/API/commands/process/detach-resumes/main.cpp new file mode 100644 index 000000000000..e8050fef2c38 --- /dev/null +++ b/lldb/test/API/commands/process/detach-resumes/main.cpp @@ -0,0 +1,48 @@ +#include "pseudo_barrier.h" +#include +#include +#include +#include +#include +#include + +pseudo_barrier_t barrier; + +constexpr size_t nthreads = 5; +volatile bool wait_for_debugger_flag = true; + +void break_here() {} + +void tfunc() { + pseudo_barrier_wait(barrier); + + break_here(); +} + +int main(int argc, char const *argv[]) { + lldb_enable_attach(); + + if (argc < 3) + return 1; + + // Create a file to signal that this process has started up. + std::ofstream(argv[1]).close(); + + // And wait for it to attach. + for (int i = 0; i < 100 && wait_for_debugger_flag; ++i) + std::this_thread::sleep_for(std::chrono::seconds(1)); + + // Fire up the threads and have them call break_here() simultaneously. + pseudo_barrier_init(barrier, nthreads); + std::vector threads; + for (size_t i = 0; i < nthreads; ++i) + threads.emplace_back(tfunc); + + for (std::thread &t : threads) + t.join(); + + // Create the file to let the debugger know we're running. + std::ofstream(argv[2]).close(); + + return 0; +} diff --git a/lldb/test/API/functionalities/fork/concurrent_vfork/TestConcurrentVFork.py b/lldb/test/API/functionalities/fork/concurrent_vfork/TestConcurrentVFork.py index 1790bd497f4e..2dcbb728549f 100644 --- a/lldb/test/API/functionalities/fork/concurrent_vfork/TestConcurrentVFork.py +++ b/lldb/test/API/functionalities/fork/concurrent_vfork/TestConcurrentVFork.py @@ -48,8 +48,6 @@ class TestConcurrentVFork(TestBase): self.expect("continue", patterns=[r"exited with status = 1[0-4]"]) @skipUnlessPlatform(["linux"]) - # See https://github.com/llvm/llvm-project/issues/85084. - @skipIf(oslist=["linux"], archs=["aarch64", "arm"]) def test_follow_parent_vfork_no_exec(self): """ Make sure that debugging concurrent vfork() from multiple threads won't crash lldb during follow-parent. @@ -58,8 +56,6 @@ class TestConcurrentVFork(TestBase): self.follow_parent_helper(use_fork=False, call_exec=False) @skipUnlessPlatform(["linux"]) - # See https://github.com/llvm/llvm-project/issues/85084. - @skipIf(oslist=["linux"], archs=["aarch64", "arm"]) def test_follow_parent_fork_no_exec(self): """ Make sure that debugging concurrent fork() from multiple threads won't crash lldb during follow-parent. @@ -68,8 +64,6 @@ class TestConcurrentVFork(TestBase): self.follow_parent_helper(use_fork=True, call_exec=False) @skipUnlessPlatform(["linux"]) - # See https://github.com/llvm/llvm-project/issues/85084. - @skipIf(oslist=["linux"], archs=["aarch64", "arm"]) def test_follow_parent_vfork_call_exec(self): """ Make sure that debugging concurrent vfork() from multiple threads won't crash lldb during follow-parent. @@ -78,8 +72,6 @@ class TestConcurrentVFork(TestBase): self.follow_parent_helper(use_fork=False, call_exec=True) @skipUnlessPlatform(["linux"]) - # See https://github.com/llvm/llvm-project/issues/85084. - @skipIf(oslist=["linux"], archs=["aarch64", "arm"]) def test_follow_parent_fork_call_exec(self): """ Make sure that debugging concurrent vfork() from multiple threads won't crash lldb during follow-parent. @@ -88,8 +80,6 @@ class TestConcurrentVFork(TestBase): self.follow_parent_helper(use_fork=True, call_exec=True) @skipUnlessPlatform(["linux"]) - # See https://github.com/llvm/llvm-project/issues/85084. - @skipIf(oslist=["linux"], archs=["aarch64", "arm"]) def test_follow_child_vfork_no_exec(self): """ Make sure that debugging concurrent vfork() from multiple threads won't crash lldb during follow-child. @@ -98,8 +88,6 @@ class TestConcurrentVFork(TestBase): self.follow_child_helper(use_fork=False, call_exec=False) @skipUnlessPlatform(["linux"]) - # See https://github.com/llvm/llvm-project/issues/85084. - @skipIf(oslist=["linux"], archs=["aarch64", "arm"]) def test_follow_child_fork_no_exec(self): """ Make sure that debugging concurrent fork() from multiple threads won't crash lldb during follow-child. @@ -108,8 +96,6 @@ class TestConcurrentVFork(TestBase): self.follow_child_helper(use_fork=True, call_exec=False) @skipUnlessPlatform(["linux"]) - # See https://github.com/llvm/llvm-project/issues/85084. - @skipIf(oslist=["linux"], archs=["aarch64", "arm"]) def test_follow_child_vfork_call_exec(self): """ Make sure that debugging concurrent vfork() from multiple threads won't crash lldb during follow-child. @@ -118,8 +104,6 @@ class TestConcurrentVFork(TestBase): self.follow_child_helper(use_fork=False, call_exec=True) @skipUnlessPlatform(["linux"]) - # See https://github.com/llvm/llvm-project/issues/85084. - @skipIf(oslist=["linux"], archs=["aarch64", "arm"]) def test_follow_child_fork_call_exec(self): """ Make sure that debugging concurrent fork() from multiple threads won't crash lldb during follow-child. -- GitLab From c8dca5bc0733e2fba81008fc33fcad1f45ba666a Mon Sep 17 00:00:00 2001 From: Sergio Afonso Date: Wed, 17 Apr 2024 12:17:50 +0100 Subject: [PATCH 036/862] [Flang][OpenMP][Lower] Refactor lowering of compound constructs (#87070) This patch simplifies the lowering from PFT to MLIR of OpenMP compound constructs (i.e. combined and composite). The new approach consists of iteratively processing the outermost leaf construct of the given combined construct until it cannot be split further. Both leaf constructs and composite ones have `gen...()` functions that are called when appropriate. This approach enables treating a leaf construct the same way regardless of if it appeared as part of a combined construct, and it also enables the lowering of composite constructs as a single unit. Previous corner cases are now handled in a more straightforward way and comments pointing to the relevant spec section are added. Directive sets are also completed with missing LOOP related constructs. --- .../flang/Semantics/openmp-directive-sets.h | 57 ++- flang/lib/Lower/OpenMP/OpenMP.cpp | 430 ++++++++++++------ 2 files changed, 333 insertions(+), 154 deletions(-) diff --git a/flang/include/flang/Semantics/openmp-directive-sets.h b/flang/include/flang/Semantics/openmp-directive-sets.h index 91773ae3ea9a..842d251b682a 100644 --- a/flang/include/flang/Semantics/openmp-directive-sets.h +++ b/flang/include/flang/Semantics/openmp-directive-sets.h @@ -32,14 +32,14 @@ static const OmpDirectiveSet topDistributeSet{ static const OmpDirectiveSet allDistributeSet{ OmpDirectiveSet{ - llvm::omp::OMPD_target_teams_distribute, - llvm::omp::OMPD_target_teams_distribute_parallel_do, - llvm::omp::OMPD_target_teams_distribute_parallel_do_simd, - llvm::omp::OMPD_target_teams_distribute_simd, - llvm::omp::OMPD_teams_distribute, - llvm::omp::OMPD_teams_distribute_parallel_do, - llvm::omp::OMPD_teams_distribute_parallel_do_simd, - llvm::omp::OMPD_teams_distribute_simd, + Directive::OMPD_target_teams_distribute, + Directive::OMPD_target_teams_distribute_parallel_do, + Directive::OMPD_target_teams_distribute_parallel_do_simd, + Directive::OMPD_target_teams_distribute_simd, + Directive::OMPD_teams_distribute, + Directive::OMPD_teams_distribute_parallel_do, + Directive::OMPD_teams_distribute_parallel_do_simd, + Directive::OMPD_teams_distribute_simd, } | topDistributeSet, }; @@ -63,10 +63,24 @@ static const OmpDirectiveSet allDoSet{ } | topDoSet, }; +static const OmpDirectiveSet topLoopSet{ + Directive::OMPD_loop, +}; + +static const OmpDirectiveSet allLoopSet{ + OmpDirectiveSet{ + Directive::OMPD_parallel_loop, + Directive::OMPD_target_parallel_loop, + Directive::OMPD_target_teams_loop, + Directive::OMPD_teams_loop, + } | topLoopSet, +}; + static const OmpDirectiveSet topParallelSet{ Directive::OMPD_parallel, Directive::OMPD_parallel_do, Directive::OMPD_parallel_do_simd, + Directive::OMPD_parallel_loop, Directive::OMPD_parallel_masked_taskloop, Directive::OMPD_parallel_masked_taskloop_simd, Directive::OMPD_parallel_master_taskloop, @@ -82,6 +96,7 @@ static const OmpDirectiveSet allParallelSet{ Directive::OMPD_target_parallel, Directive::OMPD_target_parallel_do, Directive::OMPD_target_parallel_do_simd, + Directive::OMPD_target_parallel_loop, Directive::OMPD_target_teams_distribute_parallel_do, Directive::OMPD_target_teams_distribute_parallel_do_simd, Directive::OMPD_teams_distribute_parallel_do, @@ -118,12 +133,14 @@ static const OmpDirectiveSet topTargetSet{ Directive::OMPD_target_parallel, Directive::OMPD_target_parallel_do, Directive::OMPD_target_parallel_do_simd, + Directive::OMPD_target_parallel_loop, Directive::OMPD_target_simd, Directive::OMPD_target_teams, Directive::OMPD_target_teams_distribute, Directive::OMPD_target_teams_distribute_parallel_do, Directive::OMPD_target_teams_distribute_parallel_do_simd, Directive::OMPD_target_teams_distribute_simd, + Directive::OMPD_target_teams_loop, }; static const OmpDirectiveSet allTargetSet{topTargetSet}; @@ -156,11 +173,12 @@ static const OmpDirectiveSet topTeamsSet{ static const OmpDirectiveSet allTeamsSet{ OmpDirectiveSet{ - llvm::omp::OMPD_target_teams, - llvm::omp::OMPD_target_teams_distribute, - llvm::omp::OMPD_target_teams_distribute_parallel_do, - llvm::omp::OMPD_target_teams_distribute_parallel_do_simd, - llvm::omp::OMPD_target_teams_distribute_simd, + Directive::OMPD_target_teams, + Directive::OMPD_target_teams_distribute, + Directive::OMPD_target_teams_distribute_parallel_do, + Directive::OMPD_target_teams_distribute_parallel_do_simd, + Directive::OMPD_target_teams_distribute_simd, + Directive::OMPD_target_teams_loop, } | topTeamsSet, }; @@ -178,6 +196,14 @@ static const OmpDirectiveSet allDistributeSimdSet{ static const OmpDirectiveSet allDoSimdSet{allDoSet & allSimdSet}; static const OmpDirectiveSet allTaskloopSimdSet{allTaskloopSet & allSimdSet}; +static const OmpDirectiveSet compositeConstructSet{ + Directive::OMPD_distribute_parallel_do, + Directive::OMPD_distribute_parallel_do_simd, + Directive::OMPD_distribute_simd, + Directive::OMPD_do_simd, + Directive::OMPD_taskloop_simd, +}; + static const OmpDirectiveSet blockConstructSet{ Directive::OMPD_master, Directive::OMPD_ordered, @@ -201,12 +227,14 @@ static const OmpDirectiveSet loopConstructSet{ Directive::OMPD_distribute_simd, Directive::OMPD_do, Directive::OMPD_do_simd, + Directive::OMPD_loop, Directive::OMPD_masked_taskloop, Directive::OMPD_masked_taskloop_simd, Directive::OMPD_master_taskloop, Directive::OMPD_master_taskloop_simd, Directive::OMPD_parallel_do, Directive::OMPD_parallel_do_simd, + Directive::OMPD_parallel_loop, Directive::OMPD_parallel_masked_taskloop, Directive::OMPD_parallel_masked_taskloop_simd, Directive::OMPD_parallel_master_taskloop, @@ -214,17 +242,20 @@ static const OmpDirectiveSet loopConstructSet{ Directive::OMPD_simd, Directive::OMPD_target_parallel_do, Directive::OMPD_target_parallel_do_simd, + Directive::OMPD_target_parallel_loop, Directive::OMPD_target_simd, Directive::OMPD_target_teams_distribute, Directive::OMPD_target_teams_distribute_parallel_do, Directive::OMPD_target_teams_distribute_parallel_do_simd, Directive::OMPD_target_teams_distribute_simd, + Directive::OMPD_target_teams_loop, Directive::OMPD_taskloop, Directive::OMPD_taskloop_simd, Directive::OMPD_teams_distribute, Directive::OMPD_teams_distribute_parallel_do, Directive::OMPD_teams_distribute_parallel_do_simd, Directive::OMPD_teams_distribute_simd, + Directive::OMPD_teams_loop, Directive::OMPD_tile, Directive::OMPD_unroll, }; diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp index c31d63625dbb..bb38082b245e 100644 --- a/flang/lib/Lower/OpenMP/OpenMP.cpp +++ b/flang/lib/Lower/OpenMP/OpenMP.cpp @@ -488,6 +488,81 @@ markDeclareTarget(mlir::Operation *op, declareTargetOp.setDeclareTarget(deviceType, captureClause); } +/// Split a combined directive into an outer leaf directive and the (possibly +/// combined) rest of the combined directive. Composite directives and +/// non-compound directives are not split, in which case it will return the +/// input directive as its first output and an empty value as its second output. +static std::pair> +splitCombinedDirective(llvm::omp::Directive dir) { + using D = llvm::omp::Directive; + switch (dir) { + case D::OMPD_masked_taskloop: + return {D::OMPD_masked, D::OMPD_taskloop}; + case D::OMPD_masked_taskloop_simd: + return {D::OMPD_masked, D::OMPD_taskloop_simd}; + case D::OMPD_master_taskloop: + return {D::OMPD_master, D::OMPD_taskloop}; + case D::OMPD_master_taskloop_simd: + return {D::OMPD_master, D::OMPD_taskloop_simd}; + case D::OMPD_parallel_do: + return {D::OMPD_parallel, D::OMPD_do}; + case D::OMPD_parallel_do_simd: + return {D::OMPD_parallel, D::OMPD_do_simd}; + case D::OMPD_parallel_masked: + return {D::OMPD_parallel, D::OMPD_masked}; + case D::OMPD_parallel_masked_taskloop: + return {D::OMPD_parallel, D::OMPD_masked_taskloop}; + case D::OMPD_parallel_masked_taskloop_simd: + return {D::OMPD_parallel, D::OMPD_masked_taskloop_simd}; + case D::OMPD_parallel_master: + return {D::OMPD_parallel, D::OMPD_master}; + case D::OMPD_parallel_master_taskloop: + return {D::OMPD_parallel, D::OMPD_master_taskloop}; + case D::OMPD_parallel_master_taskloop_simd: + return {D::OMPD_parallel, D::OMPD_master_taskloop_simd}; + case D::OMPD_parallel_sections: + return {D::OMPD_parallel, D::OMPD_sections}; + case D::OMPD_parallel_workshare: + return {D::OMPD_parallel, D::OMPD_workshare}; + case D::OMPD_target_parallel: + return {D::OMPD_target, D::OMPD_parallel}; + case D::OMPD_target_parallel_do: + return {D::OMPD_target, D::OMPD_parallel_do}; + case D::OMPD_target_parallel_do_simd: + return {D::OMPD_target, D::OMPD_parallel_do_simd}; + case D::OMPD_target_simd: + return {D::OMPD_target, D::OMPD_simd}; + case D::OMPD_target_teams: + return {D::OMPD_target, D::OMPD_teams}; + case D::OMPD_target_teams_distribute: + return {D::OMPD_target, D::OMPD_teams_distribute}; + case D::OMPD_target_teams_distribute_parallel_do: + return {D::OMPD_target, D::OMPD_teams_distribute_parallel_do}; + case D::OMPD_target_teams_distribute_parallel_do_simd: + return {D::OMPD_target, D::OMPD_teams_distribute_parallel_do_simd}; + case D::OMPD_target_teams_distribute_simd: + return {D::OMPD_target, D::OMPD_teams_distribute_simd}; + case D::OMPD_teams_distribute: + return {D::OMPD_teams, D::OMPD_distribute}; + case D::OMPD_teams_distribute_parallel_do: + return {D::OMPD_teams, D::OMPD_distribute_parallel_do}; + case D::OMPD_teams_distribute_parallel_do_simd: + return {D::OMPD_teams, D::OMPD_distribute_parallel_do_simd}; + case D::OMPD_teams_distribute_simd: + return {D::OMPD_teams, D::OMPD_distribute_simd}; + case D::OMPD_parallel_loop: + return {D::OMPD_parallel, D::OMPD_loop}; + case D::OMPD_target_parallel_loop: + return {D::OMPD_target, D::OMPD_parallel_loop}; + case D::OMPD_target_teams_loop: + return {D::OMPD_target, D::OMPD_teams_loop}; + case D::OMPD_teams_loop: + return {D::OMPD_teams, D::OMPD_loop}; + default: + return {dir, std::nullopt}; + } +} + //===----------------------------------------------------------------------===// // Op body generation helper structures and functions //===----------------------------------------------------------------------===// @@ -1676,7 +1751,6 @@ static OpTy genTargetEnterExitUpdateDataOp( } else { llvm_unreachable("Unexpected TARGET DATA construct"); } - mlir::omp::TargetEnterExitUpdateDataClauseOps clauseOps; genTargetEnterExitUpdateDataClauses(converter, semaCtx, stmtCtx, clauseList, loc, directive, clauseOps); @@ -1804,16 +1878,44 @@ genWsloopOp(Fortran::lower::AbstractConverter &converter, // Code generation functions for composite constructs //===----------------------------------------------------------------------===// -static void genCompositeDoSimd( +static void genCompositeDistributeParallelDo( + Fortran::lower::AbstractConverter &converter, + Fortran::semantics::SemanticsContext &semaCtx, + Fortran::lower::pft::Evaluation &eval, + const Fortran::parser::OmpClauseList &beginClauseList, + const Fortran::parser::OmpClauseList *endClauseList, mlir::Location loc) { + TODO(loc, "Composite DISTRIBUTE PARALLEL DO"); +} + +static void genCompositeDistributeParallelDoSimd( + Fortran::lower::AbstractConverter &converter, + Fortran::semantics::SemanticsContext &semaCtx, + Fortran::lower::pft::Evaluation &eval, + const Fortran::parser::OmpClauseList &beginClauseList, + const Fortran::parser::OmpClauseList *endClauseList, mlir::Location loc) { + TODO(loc, "Composite DISTRIBUTE PARALLEL DO SIMD"); +} + +static void genCompositeDistributeSimd( Fortran::lower::AbstractConverter &converter, Fortran::semantics::SemanticsContext &semaCtx, - Fortran::lower::pft::Evaluation &eval, llvm::omp::Directive ompDirective, + Fortran::lower::pft::Evaluation &eval, const Fortran::parser::OmpClauseList &beginClauseList, const Fortran::parser::OmpClauseList *endClauseList, mlir::Location loc) { + TODO(loc, "Composite DISTRIBUTE SIMD"); +} + +static void +genCompositeDoSimd(Fortran::lower::AbstractConverter &converter, + Fortran::semantics::SemanticsContext &semaCtx, + Fortran::lower::pft::Evaluation &eval, + const Fortran::parser::OmpClauseList &beginClauseList, + const Fortran::parser::OmpClauseList *endClauseList, + mlir::Location loc) { ClauseProcessor cp(converter, semaCtx, beginClauseList); cp.processTODO(loc, - ompDirective); + clause::Order, clause::Safelen, clause::Simdlen>( + loc, llvm::omp::OMPD_do_simd); // TODO: Add support for vectorization - add vectorization hints inside loop // body. // OpenMP standard does not specify the length of vector instructions. @@ -1825,6 +1927,16 @@ static void genCompositeDoSimd( genWsloopOp(converter, semaCtx, eval, loc, beginClauseList, endClauseList); } +static void +genCompositeTaskloopSimd(Fortran::lower::AbstractConverter &converter, + Fortran::semantics::SemanticsContext &semaCtx, + Fortran::lower::pft::Evaluation &eval, + const Fortran::parser::OmpClauseList &beginClauseList, + const Fortran::parser::OmpClauseList *endClauseList, + mlir::Location loc) { + TODO(loc, "Composite TASKLOOP SIMD"); +} + //===----------------------------------------------------------------------===// // OpenMPDeclarativeConstruct visitors //===----------------------------------------------------------------------===// @@ -2082,13 +2194,18 @@ genOMP(Fortran::lower::AbstractConverter &converter, std::get(blockConstruct.t); const auto &endBlockDirective = std::get(blockConstruct.t); - const auto &directive = - std::get(beginBlockDirective.t); + mlir::Location currentLocation = + converter.genLocation(beginBlockDirective.source); + const auto origDirective = + std::get(beginBlockDirective.t).v; const auto &beginClauseList = std::get(beginBlockDirective.t); const auto &endClauseList = std::get(endBlockDirective.t); + assert(llvm::omp::blockConstructSet.test(origDirective) && + "Expected block construct"); + for (const Fortran::parser::OmpClause &clause : beginClauseList.v) { mlir::Location clauseLocation = converter.genLocation(clause.source); if (!std::get_if(&clause.u) && @@ -2124,93 +2241,74 @@ genOMP(Fortran::lower::AbstractConverter &converter, TODO(clauseLocation, "OpenMP Block construct clause"); } - bool singleDirective = true; - mlir::Location currentLocation = converter.genLocation(directive.source); - switch (directive.v) { - case llvm::omp::Directive::OMPD_master: - genMasterOp(converter, semaCtx, eval, /*genNested=*/true, currentLocation); - break; - case llvm::omp::Directive::OMPD_ordered: - genOrderedRegionOp(converter, semaCtx, eval, /*genNested=*/true, - currentLocation, beginClauseList); - break; - case llvm::omp::Directive::OMPD_parallel: - genParallelOp(converter, symTable, semaCtx, eval, /*genNested=*/true, - currentLocation, beginClauseList); - break; - case llvm::omp::Directive::OMPD_single: - genSingleOp(converter, semaCtx, eval, /*genNested=*/true, currentLocation, - beginClauseList, endClauseList); - break; - case llvm::omp::Directive::OMPD_target: - genTargetOp(converter, semaCtx, eval, /*genNested=*/true, currentLocation, + std::optional nextDir = origDirective; + bool outermostLeafConstruct = true; + while (nextDir) { + llvm::omp::Directive leafDir; + std::tie(leafDir, nextDir) = splitCombinedDirective(*nextDir); + const bool genNested = !nextDir; + const bool outerCombined = outermostLeafConstruct && nextDir.has_value(); + switch (leafDir) { + case llvm::omp::Directive::OMPD_master: + // 2.16 MASTER construct. + genMasterOp(converter, semaCtx, eval, genNested, currentLocation); + break; + case llvm::omp::Directive::OMPD_ordered: + // 2.17.9 ORDERED construct. + genOrderedRegionOp(converter, semaCtx, eval, genNested, currentLocation, + beginClauseList); + break; + case llvm::omp::Directive::OMPD_parallel: + // 2.6 PARALLEL construct. + genParallelOp(converter, symTable, semaCtx, eval, genNested, + currentLocation, beginClauseList, outerCombined); + break; + case llvm::omp::Directive::OMPD_single: + // 2.8.2 SINGLE construct. + genSingleOp(converter, semaCtx, eval, genNested, currentLocation, + beginClauseList, endClauseList); + break; + case llvm::omp::Directive::OMPD_target: + // 2.12.5 TARGET construct. + genTargetOp(converter, semaCtx, eval, genNested, currentLocation, + beginClauseList, outerCombined); + break; + case llvm::omp::Directive::OMPD_target_data: + // 2.12.2 TARGET DATA construct. + genTargetDataOp(converter, semaCtx, eval, genNested, currentLocation, + beginClauseList); + break; + case llvm::omp::Directive::OMPD_task: + // 2.10.1 TASK construct. + genTaskOp(converter, semaCtx, eval, genNested, currentLocation, beginClauseList); - break; - case llvm::omp::Directive::OMPD_target_data: - genTargetDataOp(converter, semaCtx, eval, /*genNested=*/true, - currentLocation, beginClauseList); - break; - case llvm::omp::Directive::OMPD_task: - genTaskOp(converter, semaCtx, eval, /*genNested=*/true, currentLocation, - beginClauseList); - break; - case llvm::omp::Directive::OMPD_taskgroup: - genTaskgroupOp(converter, semaCtx, eval, /*genNested=*/true, - currentLocation, beginClauseList); - break; - case llvm::omp::Directive::OMPD_teams: - genTeamsOp(converter, semaCtx, eval, /*genNested=*/true, currentLocation, - beginClauseList); - break; - case llvm::omp::Directive::OMPD_workshare: - // FIXME: Workshare is not a commonly used OpenMP construct, an - // implementation for this feature will come later. For the codes - // that use this construct, add a single construct for now. - genSingleOp(converter, semaCtx, eval, /*genNested=*/true, currentLocation, - beginClauseList, endClauseList); - break; - default: - singleDirective = false; - break; - } - - if (singleDirective) - return; - - // Codegen for combined directives - bool combinedDirective = false; - if ((llvm::omp::allTargetSet & llvm::omp::blockConstructSet) - .test(directive.v)) { - genTargetOp(converter, semaCtx, eval, /*genNested=*/false, currentLocation, - beginClauseList, /*outerCombined=*/true); - combinedDirective = true; - } - if ((llvm::omp::allTeamsSet & llvm::omp::blockConstructSet) - .test(directive.v)) { - genTeamsOp(converter, semaCtx, eval, /*genNested=*/false, currentLocation, - beginClauseList); - combinedDirective = true; - } - if ((llvm::omp::allParallelSet & llvm::omp::blockConstructSet) - .test(directive.v)) { - bool outerCombined = - directive.v != llvm::omp::Directive::OMPD_target_parallel; - genParallelOp(converter, symTable, semaCtx, eval, /*genNested=*/false, - currentLocation, beginClauseList, outerCombined); - combinedDirective = true; - } - if ((llvm::omp::workShareSet & llvm::omp::blockConstructSet) - .test(directive.v)) { - genSingleOp(converter, semaCtx, eval, /*genNested=*/false, currentLocation, - beginClauseList, endClauseList); - combinedDirective = true; + break; + case llvm::omp::Directive::OMPD_taskgroup: + // 2.17.6 TASKGROUP construct. + genTaskgroupOp(converter, semaCtx, eval, genNested, currentLocation, + beginClauseList); + break; + case llvm::omp::Directive::OMPD_teams: + // 2.7 TEAMS construct. + // FIXME Pass the outerCombined argument or rename it to better describe + // what it represents if it must always be `false` in this context. + genTeamsOp(converter, semaCtx, eval, genNested, currentLocation, + beginClauseList); + break; + case llvm::omp::Directive::OMPD_workshare: + // 2.8.3 WORKSHARE construct. + // FIXME: Workshare is not a commonly used OpenMP construct, an + // implementation for this feature will come later. For the codes + // that use this construct, add a single construct for now. + genSingleOp(converter, semaCtx, eval, genNested, currentLocation, + beginClauseList, endClauseList); + break; + default: + llvm_unreachable("Unexpected block construct"); + break; + } + outermostLeafConstruct = false; } - if (!combinedDirective) - TODO(currentLocation, "Unhandled block directive (" + - llvm::omp::getOpenMPDirectiveName(directive.v) + - ")"); - - genNestedEvaluations(converter, eval); } static void @@ -2248,9 +2346,12 @@ static void genOMP(Fortran::lower::AbstractConverter &converter, std::get(beginLoopDirective.t); mlir::Location currentLocation = converter.genLocation(beginLoopDirective.source); - const auto ompDirective = + const auto origDirective = std::get(beginLoopDirective.t).v; + assert(llvm::omp::loopConstructSet.test(origDirective) && + "Expected loop construct"); + const auto *endClauseList = [&]() { using RetTy = const Fortran::parser::OmpClauseList *; if (auto &endLoopDirective = @@ -2262,56 +2363,103 @@ static void genOMP(Fortran::lower::AbstractConverter &converter, return RetTy(); }(); - bool validDirective = false; - if (llvm::omp::topTaskloopSet.test(ompDirective)) { - validDirective = true; - genTaskloopOp(converter, semaCtx, eval, currentLocation, beginClauseList); - } else { - // Create omp.{target, teams, distribute, parallel} nested operations - if ((llvm::omp::allTargetSet & llvm::omp::loopConstructSet) - .test(ompDirective)) { - validDirective = true; - genTargetOp(converter, semaCtx, eval, /*genNested=*/false, - currentLocation, beginClauseList, /*outerCombined=*/true); - } - if ((llvm::omp::allTeamsSet & llvm::omp::loopConstructSet) - .test(ompDirective)) { - validDirective = true; - genTeamsOp(converter, semaCtx, eval, /*genNested=*/false, currentLocation, - beginClauseList, /*outerCombined=*/true); - } - if (llvm::omp::allDistributeSet.test(ompDirective)) { - validDirective = true; - genDistributeOp(converter, semaCtx, eval, /*genNested=*/false, - currentLocation, beginClauseList); - } - if ((llvm::omp::allParallelSet & llvm::omp::loopConstructSet) - .test(ompDirective)) { - validDirective = true; - genParallelOp(converter, symTable, semaCtx, eval, /*genNested=*/false, - currentLocation, beginClauseList, /*outerCombined=*/true); + std::optional nextDir = origDirective; + while (nextDir) { + llvm::omp::Directive leafDir; + std::tie(leafDir, nextDir) = splitCombinedDirective(*nextDir); + if (llvm::omp::compositeConstructSet.test(leafDir)) { + assert(!nextDir && "Composite construct cannot be split"); + switch (leafDir) { + case llvm::omp::Directive::OMPD_distribute_parallel_do: + // 2.9.4.3 DISTRIBUTE PARALLEL Worksharing-Loop construct. + genCompositeDistributeParallelDo(converter, semaCtx, eval, + beginClauseList, endClauseList, + currentLocation); + break; + case llvm::omp::Directive::OMPD_distribute_parallel_do_simd: + // 2.9.4.4 DISTRIBUTE PARALLEL Worksharing-Loop SIMD construct. + genCompositeDistributeParallelDoSimd(converter, semaCtx, eval, + beginClauseList, endClauseList, + currentLocation); + break; + case llvm::omp::Directive::OMPD_distribute_simd: + // 2.9.4.2 DISTRIBUTE SIMD construct. + genCompositeDistributeSimd(converter, semaCtx, eval, beginClauseList, + endClauseList, currentLocation); + break; + case llvm::omp::Directive::OMPD_do_simd: + // 2.9.3.2 Worksharing-Loop SIMD construct. + genCompositeDoSimd(converter, semaCtx, eval, beginClauseList, + endClauseList, currentLocation); + break; + case llvm::omp::Directive::OMPD_taskloop_simd: + // 2.10.3 TASKLOOP SIMD construct. + genCompositeTaskloopSimd(converter, semaCtx, eval, beginClauseList, + endClauseList, currentLocation); + break; + default: + llvm_unreachable("Unexpected composite construct"); + } + } else { + const bool genNested = !nextDir; + switch (leafDir) { + case llvm::omp::Directive::OMPD_distribute: + // 2.9.4.1 DISTRIBUTE construct. + genDistributeOp(converter, semaCtx, eval, genNested, currentLocation, + beginClauseList); + break; + case llvm::omp::Directive::OMPD_do: + // 2.9.2 Worksharing-Loop construct. + genWsloopOp(converter, semaCtx, eval, currentLocation, beginClauseList, + endClauseList); + break; + case llvm::omp::Directive::OMPD_parallel: + // 2.6 PARALLEL construct. + // FIXME This is not necessarily always the outer leaf construct of a + // combined construct in this context (e.g. DISTRIBUTE PARALLEL DO). + // Maybe rename the argument if it represents something else or + // initialize it properly. + genParallelOp(converter, symTable, semaCtx, eval, genNested, + currentLocation, beginClauseList, + /*outerCombined=*/true); + break; + case llvm::omp::Directive::OMPD_simd: + // 2.9.3.1 SIMD construct. + genSimdOp(converter, semaCtx, eval, currentLocation, beginClauseList); + break; + case llvm::omp::Directive::OMPD_target: + // 2.12.5 TARGET construct. + genTargetOp(converter, semaCtx, eval, genNested, currentLocation, + beginClauseList, /*outerCombined=*/true); + break; + case llvm::omp::Directive::OMPD_taskloop: + // 2.10.2 TASKLOOP construct. + genTaskloopOp(converter, semaCtx, eval, currentLocation, + beginClauseList); + break; + case llvm::omp::Directive::OMPD_teams: + // 2.7 TEAMS construct. + // FIXME This is not necessarily always the outer leaf construct of a + // combined construct in this constext (e.g. TARGET TEAMS DISTRIBUTE). + // Maybe rename the argument if it represents something else or + // initialize it properly. + genTeamsOp(converter, semaCtx, eval, genNested, currentLocation, + beginClauseList, /*outerCombined=*/true); + break; + case llvm::omp::Directive::OMPD_loop: + case llvm::omp::Directive::OMPD_masked: + case llvm::omp::Directive::OMPD_master: + case llvm::omp::Directive::OMPD_tile: + case llvm::omp::Directive::OMPD_unroll: + TODO(currentLocation, "Unhandled loop directive (" + + llvm::omp::getOpenMPDirectiveName(leafDir) + + ")"); + break; + default: + llvm_unreachable("Unexpected loop construct"); + } } } - if ((llvm::omp::allDoSet | llvm::omp::allSimdSet).test(ompDirective)) - validDirective = true; - - if (!validDirective) { - TODO(currentLocation, "Unhandled loop directive (" + - llvm::omp::getOpenMPDirectiveName(ompDirective) + - ")"); - } - - if (llvm::omp::allDoSimdSet.test(ompDirective)) { - // 2.9.3.2 Workshare SIMD construct - genCompositeDoSimd(converter, semaCtx, eval, ompDirective, beginClauseList, - endClauseList, currentLocation); - } else if (llvm::omp::allSimdSet.test(ompDirective)) { - // 2.9.3.1 SIMD construct - genSimdOp(converter, semaCtx, eval, currentLocation, beginClauseList); - } else { - genWsloopOp(converter, semaCtx, eval, currentLocation, beginClauseList, - endClauseList); - } } static void -- GitLab From 06eedffe0d2782922e63cc25cb927f4acdaf7b30 Mon Sep 17 00:00:00 2001 From: NagyDonat Date: Wed, 17 Apr 2024 13:26:51 +0200 Subject: [PATCH 037/862] [analyzer] Use explicit call description mode in iterator checkers (#88913) This commit explicitly specifies the matching mode (C library function, any non-method function, or C++ method) for the `CallDescription`s constructed in the iterator/container checkers. This change won't cause major functional changes, but isn't NFC because it ensures that e.g. call descriptions for a non-method function won't accidentally match a method that has the same name. Separate commits will perform (or have already performed) this change in other checkers. My goal is to ensure that the call description mode is always explicitly specified and eliminate (or strongly restrict) the vague "may be either a method or a simple function" mode that's the current default. I'm handling the iterator checkers in this separate commit because they're infamously complex; but I don't expect any trouble because this transition doesn't interact with the "central" logic of iterator handling. --- .../Checkers/ContainerModeling.cpp | 33 +++++---- .../Checkers/DebugContainerModeling.cpp | 4 +- .../Checkers/DebugIteratorModeling.cpp | 6 +- .../Checkers/IteratorModeling.cpp | 7 +- .../Checkers/IteratorRangeChecker.cpp | 11 ++- .../Checkers/STLAlgorithmModeling.cpp | 67 ++++++++++++------- 6 files changed, 80 insertions(+), 48 deletions(-) diff --git a/clang/lib/StaticAnalyzer/Checkers/ContainerModeling.cpp b/clang/lib/StaticAnalyzer/Checkers/ContainerModeling.cpp index 009c0d3fb936..55ed809bfed6 100644 --- a/clang/lib/StaticAnalyzer/Checkers/ContainerModeling.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/ContainerModeling.cpp @@ -72,26 +72,31 @@ public: SVal) const; CallDescriptionMap NoIterParamFunctions = { - {{{"clear"}, 0}, &ContainerModeling::handleClear}, - {{{"assign"}, 2}, &ContainerModeling::handleAssign}, - {{{"push_back"}, 1}, &ContainerModeling::handlePushBack}, - {{{"emplace_back"}, 1}, &ContainerModeling::handlePushBack}, - {{{"pop_back"}, 0}, &ContainerModeling::handlePopBack}, - {{{"push_front"}, 1}, &ContainerModeling::handlePushFront}, - {{{"emplace_front"}, 1}, &ContainerModeling::handlePushFront}, - {{{"pop_front"}, 0}, &ContainerModeling::handlePopFront}, + {{CDM::CXXMethod, {"clear"}, 0}, &ContainerModeling::handleClear}, + {{CDM::CXXMethod, {"assign"}, 2}, &ContainerModeling::handleAssign}, + {{CDM::CXXMethod, {"push_back"}, 1}, &ContainerModeling::handlePushBack}, + {{CDM::CXXMethod, {"emplace_back"}, 1}, + &ContainerModeling::handlePushBack}, + {{CDM::CXXMethod, {"pop_back"}, 0}, &ContainerModeling::handlePopBack}, + {{CDM::CXXMethod, {"push_front"}, 1}, + &ContainerModeling::handlePushFront}, + {{CDM::CXXMethod, {"emplace_front"}, 1}, + &ContainerModeling::handlePushFront}, + {{CDM::CXXMethod, {"pop_front"}, 0}, &ContainerModeling::handlePopFront}, }; CallDescriptionMap OneIterParamFunctions = { - {{{"insert"}, 2}, &ContainerModeling::handleInsert}, - {{{"emplace"}, 2}, &ContainerModeling::handleInsert}, - {{{"erase"}, 1}, &ContainerModeling::handleErase}, - {{{"erase_after"}, 1}, &ContainerModeling::handleEraseAfter}, + {{CDM::CXXMethod, {"insert"}, 2}, &ContainerModeling::handleInsert}, + {{CDM::CXXMethod, {"emplace"}, 2}, &ContainerModeling::handleInsert}, + {{CDM::CXXMethod, {"erase"}, 1}, &ContainerModeling::handleErase}, + {{CDM::CXXMethod, {"erase_after"}, 1}, + &ContainerModeling::handleEraseAfter}, }; CallDescriptionMap TwoIterParamFunctions = { - {{{"erase"}, 2}, &ContainerModeling::handleErase}, - {{{"erase_after"}, 2}, &ContainerModeling::handleEraseAfter}, + {{CDM::CXXMethod, {"erase"}, 2}, &ContainerModeling::handleErase}, + {{CDM::CXXMethod, {"erase_after"}, 2}, + &ContainerModeling::handleEraseAfter}, }; }; diff --git a/clang/lib/StaticAnalyzer/Checkers/DebugContainerModeling.cpp b/clang/lib/StaticAnalyzer/Checkers/DebugContainerModeling.cpp index 72186a99d943..d3830a01dd0c 100644 --- a/clang/lib/StaticAnalyzer/Checkers/DebugContainerModeling.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/DebugContainerModeling.cpp @@ -42,9 +42,9 @@ class DebugContainerModeling CheckerContext &) const; CallDescriptionMap Callbacks = { - {{{"clang_analyzer_container_begin"}, 1}, + {{CDM::SimpleFunc, {"clang_analyzer_container_begin"}, 1}, &DebugContainerModeling::analyzerContainerBegin}, - {{{"clang_analyzer_container_end"}, 1}, + {{CDM::SimpleFunc, {"clang_analyzer_container_end"}, 1}, &DebugContainerModeling::analyzerContainerEnd}, }; diff --git a/clang/lib/StaticAnalyzer/Checkers/DebugIteratorModeling.cpp b/clang/lib/StaticAnalyzer/Checkers/DebugIteratorModeling.cpp index 79ab71d7829d..203743dacda6 100644 --- a/clang/lib/StaticAnalyzer/Checkers/DebugIteratorModeling.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/DebugIteratorModeling.cpp @@ -43,11 +43,11 @@ class DebugIteratorModeling CheckerContext &) const; CallDescriptionMap Callbacks = { - {{{"clang_analyzer_iterator_position"}, 1}, + {{CDM::SimpleFunc, {"clang_analyzer_iterator_position"}, 1}, &DebugIteratorModeling::analyzerIteratorPosition}, - {{{"clang_analyzer_iterator_container"}, 1}, + {{CDM::SimpleFunc, {"clang_analyzer_iterator_container"}, 1}, &DebugIteratorModeling::analyzerIteratorContainer}, - {{{"clang_analyzer_iterator_validity"}, 1}, + {{CDM::SimpleFunc, {"clang_analyzer_iterator_validity"}, 1}, &DebugIteratorModeling::analyzerIteratorValidity}, }; diff --git a/clang/lib/StaticAnalyzer/Checkers/IteratorModeling.cpp b/clang/lib/StaticAnalyzer/Checkers/IteratorModeling.cpp index a95e811c2a41..5649454b4cd4 100644 --- a/clang/lib/StaticAnalyzer/Checkers/IteratorModeling.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/IteratorModeling.cpp @@ -129,19 +129,20 @@ class IteratorModeling CallDescriptionMap AdvanceLikeFunctions = { // template // void advance(InputIt& it, Distance n); - {{{"std", "advance"}, 2}, &IteratorModeling::handleAdvance}, + {{CDM::SimpleFunc, {"std", "advance"}, 2}, + &IteratorModeling::handleAdvance}, // template // BidirIt prev( // BidirIt it, // typename std::iterator_traits::difference_type n = 1); - {{{"std", "prev"}, 2}, &IteratorModeling::handlePrev}, + {{CDM::SimpleFunc, {"std", "prev"}, 2}, &IteratorModeling::handlePrev}, // template // ForwardIt next( // ForwardIt it, // typename std::iterator_traits::difference_type n = 1); - {{{"std", "next"}, 2}, &IteratorModeling::handleNext}, + {{CDM::SimpleFunc, {"std", "next"}, 2}, &IteratorModeling::handleNext}, }; public: diff --git a/clang/lib/StaticAnalyzer/Checkers/IteratorRangeChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/IteratorRangeChecker.cpp index d2b61fb92483..4dd2f700a2a0 100644 --- a/clang/lib/StaticAnalyzer/Checkers/IteratorRangeChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/IteratorRangeChecker.cpp @@ -56,10 +56,15 @@ public: using AdvanceFn = void (IteratorRangeChecker::*)(CheckerContext &, SVal, SVal) const; + // FIXME: these three functions are also listed in IteratorModeling.cpp, + // perhaps unify their handling? CallDescriptionMap AdvanceFunctions = { - {{{"std", "advance"}, 2}, &IteratorRangeChecker::verifyAdvance}, - {{{"std", "prev"}, 2}, &IteratorRangeChecker::verifyPrev}, - {{{"std", "next"}, 2}, &IteratorRangeChecker::verifyNext}, + {{CDM::SimpleFunc, {"std", "advance"}, 2}, + &IteratorRangeChecker::verifyAdvance}, + {{CDM::SimpleFunc, {"std", "prev"}, 2}, + &IteratorRangeChecker::verifyPrev}, + {{CDM::SimpleFunc, {"std", "next"}, 2}, + &IteratorRangeChecker::verifyNext}, }; }; diff --git a/clang/lib/StaticAnalyzer/Checkers/STLAlgorithmModeling.cpp b/clang/lib/StaticAnalyzer/Checkers/STLAlgorithmModeling.cpp index a5173a05636a..e037719b9029 100644 --- a/clang/lib/StaticAnalyzer/Checkers/STLAlgorithmModeling.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/STLAlgorithmModeling.cpp @@ -33,29 +33,50 @@ class STLAlgorithmModeling : public Checker { const CallExpr *) const; const CallDescriptionMap Callbacks = { - {{{"std", "find"}, 3}, &STLAlgorithmModeling::evalFind}, - {{{"std", "find"}, 4}, &STLAlgorithmModeling::evalFind}, - {{{"std", "find_if"}, 3}, &STLAlgorithmModeling::evalFind}, - {{{"std", "find_if"}, 4}, &STLAlgorithmModeling::evalFind}, - {{{"std", "find_if_not"}, 3}, &STLAlgorithmModeling::evalFind}, - {{{"std", "find_if_not"}, 4}, &STLAlgorithmModeling::evalFind}, - {{{"std", "find_first_of"}, 4}, &STLAlgorithmModeling::evalFind}, - {{{"std", "find_first_of"}, 5}, &STLAlgorithmModeling::evalFind}, - {{{"std", "find_first_of"}, 6}, &STLAlgorithmModeling::evalFind}, - {{{"std", "find_end"}, 4}, &STLAlgorithmModeling::evalFind}, - {{{"std", "find_end"}, 5}, &STLAlgorithmModeling::evalFind}, - {{{"std", "find_end"}, 6}, &STLAlgorithmModeling::evalFind}, - {{{"std", "lower_bound"}, 3}, &STLAlgorithmModeling::evalFind}, - {{{"std", "lower_bound"}, 4}, &STLAlgorithmModeling::evalFind}, - {{{"std", "upper_bound"}, 3}, &STLAlgorithmModeling::evalFind}, - {{{"std", "upper_bound"}, 4}, &STLAlgorithmModeling::evalFind}, - {{{"std", "search"}, 3}, &STLAlgorithmModeling::evalFind}, - {{{"std", "search"}, 4}, &STLAlgorithmModeling::evalFind}, - {{{"std", "search"}, 5}, &STLAlgorithmModeling::evalFind}, - {{{"std", "search"}, 6}, &STLAlgorithmModeling::evalFind}, - {{{"std", "search_n"}, 4}, &STLAlgorithmModeling::evalFind}, - {{{"std", "search_n"}, 5}, &STLAlgorithmModeling::evalFind}, - {{{"std", "search_n"}, 6}, &STLAlgorithmModeling::evalFind}, + {{CDM::SimpleFunc, {"std", "find"}, 3}, &STLAlgorithmModeling::evalFind}, + {{CDM::SimpleFunc, {"std", "find"}, 4}, &STLAlgorithmModeling::evalFind}, + {{CDM::SimpleFunc, {"std", "find_if"}, 3}, + &STLAlgorithmModeling::evalFind}, + {{CDM::SimpleFunc, {"std", "find_if"}, 4}, + &STLAlgorithmModeling::evalFind}, + {{CDM::SimpleFunc, {"std", "find_if_not"}, 3}, + &STLAlgorithmModeling::evalFind}, + {{CDM::SimpleFunc, {"std", "find_if_not"}, 4}, + &STLAlgorithmModeling::evalFind}, + {{CDM::SimpleFunc, {"std", "find_first_of"}, 4}, + &STLAlgorithmModeling::evalFind}, + {{CDM::SimpleFunc, {"std", "find_first_of"}, 5}, + &STLAlgorithmModeling::evalFind}, + {{CDM::SimpleFunc, {"std", "find_first_of"}, 6}, + &STLAlgorithmModeling::evalFind}, + {{CDM::SimpleFunc, {"std", "find_end"}, 4}, + &STLAlgorithmModeling::evalFind}, + {{CDM::SimpleFunc, {"std", "find_end"}, 5}, + &STLAlgorithmModeling::evalFind}, + {{CDM::SimpleFunc, {"std", "find_end"}, 6}, + &STLAlgorithmModeling::evalFind}, + {{CDM::SimpleFunc, {"std", "lower_bound"}, 3}, + &STLAlgorithmModeling::evalFind}, + {{CDM::SimpleFunc, {"std", "lower_bound"}, 4}, + &STLAlgorithmModeling::evalFind}, + {{CDM::SimpleFunc, {"std", "upper_bound"}, 3}, + &STLAlgorithmModeling::evalFind}, + {{CDM::SimpleFunc, {"std", "upper_bound"}, 4}, + &STLAlgorithmModeling::evalFind}, + {{CDM::SimpleFunc, {"std", "search"}, 3}, + &STLAlgorithmModeling::evalFind}, + {{CDM::SimpleFunc, {"std", "search"}, 4}, + &STLAlgorithmModeling::evalFind}, + {{CDM::SimpleFunc, {"std", "search"}, 5}, + &STLAlgorithmModeling::evalFind}, + {{CDM::SimpleFunc, {"std", "search"}, 6}, + &STLAlgorithmModeling::evalFind}, + {{CDM::SimpleFunc, {"std", "search_n"}, 4}, + &STLAlgorithmModeling::evalFind}, + {{CDM::SimpleFunc, {"std", "search_n"}, 5}, + &STLAlgorithmModeling::evalFind}, + {{CDM::SimpleFunc, {"std", "search_n"}, 6}, + &STLAlgorithmModeling::evalFind}, }; public: -- GitLab From 631c5e818ef8bb0f61fd3bb44cc4449be2142e2b Mon Sep 17 00:00:00 2001 From: MbjYjbpivj Date: Wed, 17 Apr 2024 20:15:30 +0800 Subject: [PATCH 038/862] [mlir] fix intNEQValue summary (#89029) Fix the summary of intNEQValue. --- mlir/include/mlir/IR/CommonAttrConstraints.td | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlir/include/mlir/IR/CommonAttrConstraints.td b/mlir/include/mlir/IR/CommonAttrConstraints.td index 0312ac7ec1d8..0d69bb0717a5 100644 --- a/mlir/include/mlir/IR/CommonAttrConstraints.td +++ b/mlir/include/mlir/IR/CommonAttrConstraints.td @@ -755,7 +755,7 @@ class AllAttrOf constraints> : AttrConstraint< class IntNEQValue : AttrConstraint< CPred<"::llvm::cast<::mlir::IntegerAttr>($_self).getInt() != " # n>, - "whose minimum value is " # n>; + "whose value is not " # n>; class IntMinValue : AttrConstraint< CPred<"::llvm::cast<::mlir::IntegerAttr>($_self).getInt() >= " # n>, -- GitLab From d57907d0b4f292f148310695ed011fe5a0585d6b Mon Sep 17 00:00:00 2001 From: Louis Dionne Date: Wed, 17 Apr 2024 08:21:48 -0400 Subject: [PATCH 039/862] [libc++] Add missing iterator requirement checks in the PSTL (#88127) Also add tests for those, and add a few missing requirements to testing iterators in the test suite. --- .../__algorithm/pstl_any_all_none_of.h | 6 +- libcxx/include/__algorithm/pstl_copy.h | 13 ++ libcxx/include/__algorithm/pstl_count.h | 5 + libcxx/include/__algorithm/pstl_equal.h | 9 + libcxx/include/__algorithm/pstl_fill.h | 6 +- libcxx/include/__algorithm/pstl_find.h | 6 +- libcxx/include/__algorithm/pstl_for_each.h | 4 +- libcxx/include/__algorithm/pstl_generate.h | 5 +- .../include/__algorithm/pstl_is_partitioned.h | 2 + libcxx/include/__algorithm/pstl_merge.h | 5 + libcxx/include/__algorithm/pstl_move.h | 5 + libcxx/include/__algorithm/pstl_replace.h | 13 ++ libcxx/include/__algorithm/pstl_rotate_copy.h | 5 + libcxx/include/__algorithm/pstl_sort.h | 3 + libcxx/include/__algorithm/pstl_stable_sort.h | 2 + libcxx/include/__algorithm/pstl_transform.h | 16 +- .../__iterator/cpp17_iterator_concepts.h | 38 ++-- libcxx/include/__numeric/pstl_reduce.h | 3 + .../include/__numeric/pstl_transform_reduce.h | 6 + .../cpp17_iterator_concepts.verify.cpp | 125 ++++++------ .../pstl.iterator-requirements.verify.cpp | 192 ++++++++++++++++++ libcxx/test/support/test_iterators.h | 16 +- 22 files changed, 383 insertions(+), 102 deletions(-) create mode 100644 libcxx/test/libcxx/algorithms/pstl.iterator-requirements.verify.cpp diff --git a/libcxx/include/__algorithm/pstl_any_all_none_of.h b/libcxx/include/__algorithm/pstl_any_all_none_of.h index 4b1e0e61b542..911a7e42b3fa 100644 --- a/libcxx/include/__algorithm/pstl_any_all_none_of.h +++ b/libcxx/include/__algorithm/pstl_any_all_none_of.h @@ -60,7 +60,7 @@ template , int> = 0> _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI bool any_of(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "any_of requires a ForwardIterator"); auto __res = std::__any_of(__policy, std::move(__first), std::move(__last), std::move(__pred)); if (!__res) std::__throw_bad_alloc(); @@ -99,7 +99,7 @@ template , int> = 0> _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI bool all_of(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Pred __pred) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "all_of requires a ForwardIterator"); auto __res = std::__all_of(__policy, std::move(__first), std::move(__last), std::move(__pred)); if (!__res) std::__throw_bad_alloc(); @@ -136,7 +136,7 @@ template , int> = 0> _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI bool none_of(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Pred __pred) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "none_of requires a ForwardIterator"); auto __res = std::__none_of(__policy, std::move(__first), std::move(__last), std::move(__pred)); if (!__res) std::__throw_bad_alloc(); diff --git a/libcxx/include/__algorithm/pstl_copy.h b/libcxx/include/__algorithm/pstl_copy.h index 1069dcec0e11..f35bb9713ef1 100644 --- a/libcxx/include/__algorithm/pstl_copy.h +++ b/libcxx/include/__algorithm/pstl_copy.h @@ -16,6 +16,7 @@ #include <__config> #include <__functional/identity.h> #include <__iterator/concepts.h> +#include <__iterator/cpp17_iterator_concepts.h> #include <__type_traits/enable_if.h> #include <__type_traits/is_constant_evaluated.h> #include <__type_traits/is_execution_policy.h> @@ -67,6 +68,12 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI _ForwardOutIterator copy(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _ForwardOutIterator __result) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR( + _ForwardIterator, "copy(first, last, result) requires [first, last) to be ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR( + _ForwardOutIterator, "copy(first, last, result) requires result to be a ForwardIterator"); + _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR( + _ForwardOutIterator, decltype(*__first), "copy(first, last, result) requires result to be an OutputIterator"); auto __res = std::__copy(__policy, std::move(__first), std::move(__last), std::move(__result)); if (!__res) std::__throw_bad_alloc(); @@ -106,6 +113,12 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI _ForwardOutIterator copy_n(_ExecutionPolicy&& __policy, _ForwardIterator __first, _Size __n, _ForwardOutIterator __result) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR( + _ForwardIterator, "copy_n(first, n, result) requires first to be a ForwardIterator"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR( + _ForwardOutIterator, "copy_n(first, n, result) requires result to be a ForwardIterator"); + _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR( + _ForwardOutIterator, decltype(*__first), "copy_n(first, n, result) requires result to be an OutputIterator"); auto __res = std::__copy_n(__policy, std::move(__first), std::move(__n), std::move(__result)); if (!__res) std::__throw_bad_alloc(); diff --git a/libcxx/include/__algorithm/pstl_count.h b/libcxx/include/__algorithm/pstl_count.h index 2781f6bfd3c9..6ff57cac334e 100644 --- a/libcxx/include/__algorithm/pstl_count.h +++ b/libcxx/include/__algorithm/pstl_count.h @@ -17,6 +17,7 @@ #include <__atomic/atomic.h> #include <__config> #include <__functional/operations.h> +#include <__iterator/cpp17_iterator_concepts.h> #include <__iterator/iterator_traits.h> #include <__numeric/pstl_transform_reduce.h> #include <__type_traits/enable_if.h> @@ -70,6 +71,8 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI __iter_diff_t<_ForwardIterator> count_if(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR( + _ForwardIterator, "count_if(first, last, pred) requires [first, last) to be ForwardIterators"); auto __res = std::__count_if(__policy, std::move(__first), std::move(__last), std::move(__pred)); if (!__res) std::__throw_bad_alloc(); @@ -106,6 +109,8 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI __iter_diff_t<_ForwardIterator> count(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR( + _ForwardIterator, "count(first, last, val) requires [first, last) to be ForwardIterators"); auto __res = std::__count(__policy, std::move(__first), std::move(__last), __value); if (!__res) std::__throw_bad_alloc(); diff --git a/libcxx/include/__algorithm/pstl_equal.h b/libcxx/include/__algorithm/pstl_equal.h index d235c0f4f419..0b38197d7f63 100644 --- a/libcxx/include/__algorithm/pstl_equal.h +++ b/libcxx/include/__algorithm/pstl_equal.h @@ -13,6 +13,7 @@ #include <__algorithm/pstl_frontend_dispatch.h> #include <__config> #include <__functional/operations.h> +#include <__iterator/cpp17_iterator_concepts.h> #include <__iterator/iterator_traits.h> #include <__numeric/pstl_transform_reduce.h> #include <__utility/move.h> @@ -74,6 +75,8 @@ equal(_ExecutionPolicy&& __policy, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _Pred __pred) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1, "equal requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2, "equal requires ForwardIterators"); auto __res = std::__equal(__policy, std::move(__first1), std::move(__last1), std::move(__first2), std::move(__pred)); if (!__res) std::__throw_bad_alloc(); @@ -86,6 +89,8 @@ template >, int> = 0> _LIBCPP_HIDE_FROM_ABI bool equal(_ExecutionPolicy&& __policy, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1, "equal requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2, "equal requires ForwardIterators"); return std::equal(__policy, std::move(__first1), std::move(__last1), std::move(__first2), std::equal_to{}); } @@ -145,6 +150,8 @@ equal(_ExecutionPolicy&& __policy, _ForwardIterator2 __first2, _ForwardIterator2 __last2, _Pred __pred) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1, "equal requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2, "equal requires ForwardIterators"); auto __res = std::__equal( __policy, std::move(__first1), std::move(__last1), std::move(__first2), std::move(__last2), std::move(__pred)); if (!__res) @@ -162,6 +169,8 @@ equal(_ExecutionPolicy&& __policy, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _ForwardIterator2 __last2) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1, "equal requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2, "equal requires ForwardIterators"); return std::equal( __policy, std::move(__first1), std::move(__last1), std::move(__first2), std::move(__last2), std::equal_to{}); } diff --git a/libcxx/include/__algorithm/pstl_fill.h b/libcxx/include/__algorithm/pstl_fill.h index 488b49a0feec..fd248506bc4b 100644 --- a/libcxx/include/__algorithm/pstl_fill.h +++ b/libcxx/include/__algorithm/pstl_fill.h @@ -43,7 +43,6 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI optional<__empty> __fill(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) noexcept { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator); return std::__pstl_frontend_dispatch( _LIBCPP_PSTL_CUSTOMIZATION_POINT(__pstl_fill, _RawPolicy), [&](_ForwardIterator __g_first, _ForwardIterator __g_last, const _Tp& __g_value) { @@ -63,7 +62,7 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI void fill(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "fill requires ForwardIterators"); if (!std::__fill(__policy, std::move(__first), std::move(__last), __value)) std::__throw_bad_alloc(); } @@ -79,7 +78,6 @@ template , int> = 0> [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty> __fill_n(_ExecutionPolicy&& __policy, _ForwardIterator&& __first, _SizeT&& __n, const _Tp& __value) noexcept { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator); return std::__pstl_frontend_dispatch( _LIBCPP_PSTL_CUSTOMIZATION_POINT(__pstl_fill_n, _RawPolicy), [&](_ForwardIterator __g_first, _SizeT __g_n, const _Tp& __g_value) { @@ -102,7 +100,7 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI void fill_n(_ExecutionPolicy&& __policy, _ForwardIterator __first, _SizeT __n, const _Tp& __value) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "fill_n requires ForwardIterators"); if (!std::__fill_n(__policy, std::move(__first), std::move(__n), __value)) std::__throw_bad_alloc(); } diff --git a/libcxx/include/__algorithm/pstl_find.h b/libcxx/include/__algorithm/pstl_find.h index 5b694db68aea..3b30a7bc9b45 100644 --- a/libcxx/include/__algorithm/pstl_find.h +++ b/libcxx/include/__algorithm/pstl_find.h @@ -50,7 +50,7 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI _ForwardIterator find_if(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "find_if requires ForwardIterators"); auto __res = std::__find_if(__policy, std::move(__first), std::move(__last), std::move(__pred)); if (!__res) std::__throw_bad_alloc(); @@ -88,7 +88,7 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI _ForwardIterator find_if_not(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "find_if_not requires ForwardIterators"); auto __res = std::__find_if_not(__policy, std::move(__first), std::move(__last), std::move(__pred)); if (!__res) std::__throw_bad_alloc(); @@ -125,7 +125,7 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI _ForwardIterator find(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "find requires ForwardIterators"); auto __res = std::__find(__policy, std::move(__first), std::move(__last), __value); if (!__res) std::__throw_bad_alloc(); diff --git a/libcxx/include/__algorithm/pstl_for_each.h b/libcxx/include/__algorithm/pstl_for_each.h index bb7b5a61a6dc..a9ebed74a62f 100644 --- a/libcxx/include/__algorithm/pstl_for_each.h +++ b/libcxx/include/__algorithm/pstl_for_each.h @@ -53,7 +53,7 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI void for_each(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Function __func) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "for_each requires ForwardIterators"); if (!std::__for_each(__policy, std::move(__first), std::move(__last), std::move(__func))) std::__throw_bad_alloc(); } @@ -93,7 +93,7 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI void for_each_n(_ExecutionPolicy&& __policy, _ForwardIterator __first, _Size __size, _Function __func) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "for_each_n requires a ForwardIterator"); auto __res = std::__for_each_n(__policy, std::move(__first), std::move(__size), std::move(__func)); if (!__res) std::__throw_bad_alloc(); diff --git a/libcxx/include/__algorithm/pstl_generate.h b/libcxx/include/__algorithm/pstl_generate.h index 7133c6f4f4c6..886af290d7f2 100644 --- a/libcxx/include/__algorithm/pstl_generate.h +++ b/libcxx/include/__algorithm/pstl_generate.h @@ -42,7 +42,6 @@ template , int> = 0> [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty> __generate(_ExecutionPolicy&& __policy, _ForwardIterator&& __first, _ForwardIterator&& __last, _Generator&& __gen) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator); return std::__pstl_frontend_dispatch( _LIBCPP_PSTL_CUSTOMIZATION_POINT(__pstl_generate, _RawPolicy), [&__policy](_ForwardIterator __g_first, _ForwardIterator __g_last, _Generator __g_gen) { @@ -63,7 +62,7 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI void generate(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Generator __gen) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "generate requires ForwardIterators"); if (!std::__generate(__policy, std::move(__first), std::move(__last), std::move(__gen))) std::__throw_bad_alloc(); } @@ -100,7 +99,7 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI void generate_n(_ExecutionPolicy&& __policy, _ForwardIterator __first, _Size __n, _Generator __gen) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "generate_n requires a ForwardIterator"); if (!std::__generate_n(__policy, std::move(__first), std::move(__n), std::move(__gen))) std::__throw_bad_alloc(); } diff --git a/libcxx/include/__algorithm/pstl_is_partitioned.h b/libcxx/include/__algorithm/pstl_is_partitioned.h index b65430212207..108bb1e43252 100644 --- a/libcxx/include/__algorithm/pstl_is_partitioned.h +++ b/libcxx/include/__algorithm/pstl_is_partitioned.h @@ -14,6 +14,7 @@ #include <__algorithm/pstl_find.h> #include <__algorithm/pstl_frontend_dispatch.h> #include <__config> +#include <__iterator/cpp17_iterator_concepts.h> #include <__type_traits/enable_if.h> #include <__type_traits/is_execution_policy.h> #include <__type_traits/remove_cvref.h> @@ -62,6 +63,7 @@ template , int> = 0> _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI bool is_partitioned(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "is_partitioned requires ForwardIterators"); auto __res = std::__is_partitioned(__policy, std::move(__first), std::move(__last), std::move(__pred)); if (!__res) std::__throw_bad_alloc(); diff --git a/libcxx/include/__algorithm/pstl_merge.h b/libcxx/include/__algorithm/pstl_merge.h index 3d262db6bc0c..d03cd8c7fbd5 100644 --- a/libcxx/include/__algorithm/pstl_merge.h +++ b/libcxx/include/__algorithm/pstl_merge.h @@ -12,6 +12,7 @@ #include <__algorithm/pstl_backend.h> #include <__config> #include <__functional/operations.h> +#include <__iterator/cpp17_iterator_concepts.h> #include <__type_traits/enable_if.h> #include <__type_traits/is_execution_policy.h> #include <__type_traits/remove_cvref.h> @@ -70,6 +71,10 @@ merge(_ExecutionPolicy&& __policy, _ForwardIterator2 __last2, _ForwardOutIterator __result, _Comp __comp = {}) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1, "merge requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2, "merge requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(_ForwardOutIterator, decltype(*__first1), "merge requires an OutputIterator"); + _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(_ForwardOutIterator, decltype(*__first2), "merge requires an OutputIterator"); auto __res = std::__merge( __policy, std::move(__first1), diff --git a/libcxx/include/__algorithm/pstl_move.h b/libcxx/include/__algorithm/pstl_move.h index d8441f1a6c2e..f4c8c1fbb2e8 100644 --- a/libcxx/include/__algorithm/pstl_move.h +++ b/libcxx/include/__algorithm/pstl_move.h @@ -15,6 +15,7 @@ #include <__algorithm/pstl_transform.h> #include <__config> #include <__functional/identity.h> +#include <__iterator/cpp17_iterator_concepts.h> #include <__iterator/iterator_traits.h> #include <__type_traits/enable_if.h> #include <__type_traits/is_constant_evaluated.h> @@ -69,6 +70,10 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI _ForwardOutIterator move(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _ForwardOutIterator __result) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "move requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardOutIterator, "move requires an OutputIterator"); + _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR( + _ForwardOutIterator, decltype(std::move(*__first)), "move requires an OutputIterator"); auto __res = std::__move(__policy, std::move(__first), std::move(__last), std::move(__result)); if (!__res) std::__throw_bad_alloc(); diff --git a/libcxx/include/__algorithm/pstl_replace.h b/libcxx/include/__algorithm/pstl_replace.h index b1caf3fd4ac0..73ac11cda26a 100644 --- a/libcxx/include/__algorithm/pstl_replace.h +++ b/libcxx/include/__algorithm/pstl_replace.h @@ -14,6 +14,7 @@ #include <__algorithm/pstl_frontend_dispatch.h> #include <__algorithm/pstl_transform.h> #include <__config> +#include <__iterator/cpp17_iterator_concepts.h> #include <__iterator/iterator_traits.h> #include <__type_traits/enable_if.h> #include <__type_traits/remove_cvref.h> @@ -74,6 +75,7 @@ replace_if(_ExecutionPolicy&& __policy, _ForwardIterator __last, _Pred __pred, const _Tp& __new_value) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "replace_if requires ForwardIterators"); auto __res = std::__replace_if(__policy, std::move(__first), std::move(__last), std::move(__pred), __new_value); if (!__res) std::__throw_bad_alloc(); @@ -121,6 +123,7 @@ replace(_ExecutionPolicy&& __policy, _ForwardIterator __last, const _Tp& __old_value, const _Tp& __new_value) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "replace requires ForwardIterators"); if (!std::__replace(__policy, std::move(__first), std::move(__last), __old_value, __new_value)) std::__throw_bad_alloc(); } @@ -177,6 +180,11 @@ _LIBCPP_HIDE_FROM_ABI void replace_copy_if( _ForwardOutIterator __result, _Pred __pred, const _Tp& __new_value) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "replace_copy_if requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardOutIterator, "replace_copy_if requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR( + _ForwardOutIterator, decltype(*__first), "replace_copy_if requires an OutputIterator"); + _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(_ForwardOutIterator, const _Tp&, "replace_copy requires an OutputIterator"); if (!std::__replace_copy_if( __policy, std::move(__first), std::move(__last), std::move(__result), std::move(__pred), __new_value)) std::__throw_bad_alloc(); @@ -233,6 +241,11 @@ _LIBCPP_HIDE_FROM_ABI void replace_copy( _ForwardOutIterator __result, const _Tp& __old_value, const _Tp& __new_value) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "replace_copy requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardOutIterator, "replace_copy requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR( + _ForwardOutIterator, decltype(*__first), "replace_copy requires an OutputIterator"); + _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(_ForwardOutIterator, const _Tp&, "replace_copy requires an OutputIterator"); if (!std::__replace_copy( __policy, std::move(__first), std::move(__last), std::move(__result), __old_value, __new_value)) std::__throw_bad_alloc(); diff --git a/libcxx/include/__algorithm/pstl_rotate_copy.h b/libcxx/include/__algorithm/pstl_rotate_copy.h index 346aab1d4a55..adab3958fe31 100644 --- a/libcxx/include/__algorithm/pstl_rotate_copy.h +++ b/libcxx/include/__algorithm/pstl_rotate_copy.h @@ -12,6 +12,7 @@ #include <__algorithm/pstl_backend.h> #include <__algorithm/pstl_copy.h> #include <__algorithm/pstl_frontend_dispatch.h> +#include <__iterator/cpp17_iterator_concepts.h> #include <__type_traits/is_execution_policy.h> #include @@ -69,6 +70,10 @@ _LIBCPP_HIDE_FROM_ABI _ForwardOutIterator rotate_copy( _ForwardIterator __middle, _ForwardIterator __last, _ForwardOutIterator __result) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "rotate_copy requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardOutIterator, "rotate_copy requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR( + _ForwardOutIterator, decltype(*__first), "rotate_copy requires an OutputIterator"); auto __res = std::__rotate_copy(__policy, std::move(__first), std::move(__middle), std::move(__last), std::move(__result)); if (!__res) diff --git a/libcxx/include/__algorithm/pstl_sort.h b/libcxx/include/__algorithm/pstl_sort.h index a931f768111a..65bc794ca6f4 100644 --- a/libcxx/include/__algorithm/pstl_sort.h +++ b/libcxx/include/__algorithm/pstl_sort.h @@ -14,6 +14,7 @@ #include <__algorithm/pstl_stable_sort.h> #include <__config> #include <__functional/operations.h> +#include <__iterator/cpp17_iterator_concepts.h> #include <__type_traits/is_execution_policy.h> #include <__type_traits/remove_cvref.h> #include <__utility/empty.h> @@ -60,6 +61,7 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI void sort(_ExecutionPolicy&& __policy, _RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp) { + _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(_RandomAccessIterator, "sort requires RandomAccessIterators"); if (!std::__sort(__policy, std::move(__first), std::move(__last), std::move(__comp))) std::__throw_bad_alloc(); } @@ -70,6 +72,7 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI void sort(_ExecutionPolicy&& __policy, _RandomAccessIterator __first, _RandomAccessIterator __last) { + _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(_RandomAccessIterator, "sort requires RandomAccessIterators"); std::sort(std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), less{}); } diff --git a/libcxx/include/__algorithm/pstl_stable_sort.h b/libcxx/include/__algorithm/pstl_stable_sort.h index 8ea0bb3f9a8d..79b94557e3dc 100644 --- a/libcxx/include/__algorithm/pstl_stable_sort.h +++ b/libcxx/include/__algorithm/pstl_stable_sort.h @@ -12,6 +12,7 @@ #include <__algorithm/pstl_backend.h> #include <__config> #include <__functional/operations.h> +#include <__iterator/cpp17_iterator_concepts.h> #include <__type_traits/enable_if.h> #include <__type_traits/is_execution_policy.h> #include <__type_traits/remove_cvref.h> @@ -48,6 +49,7 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI void stable_sort( _ExecutionPolicy&& __policy, _RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp = {}) { + _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(_RandomAccessIterator, "stable_sort requires RandomAccessIterators"); if (!std::__stable_sort(__policy, std::move(__first), std::move(__last), std::move(__comp))) std::__throw_bad_alloc(); } diff --git a/libcxx/include/__algorithm/pstl_transform.h b/libcxx/include/__algorithm/pstl_transform.h index f95938782fc3..a01a64a43cf1 100644 --- a/libcxx/include/__algorithm/pstl_transform.h +++ b/libcxx/include/__algorithm/pstl_transform.h @@ -58,9 +58,10 @@ _LIBCPP_HIDE_FROM_ABI _ForwardOutIterator transform( _ForwardIterator __last, _ForwardOutIterator __result, _UnaryOperation __op) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator); - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardOutIterator); - _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(_ForwardOutIterator, decltype(__op(*__first))); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "transform requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardOutIterator, "transform requires an OutputIterator"); + _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR( + _ForwardOutIterator, decltype(__op(*__first)), "transform requires an OutputIterator"); auto __res = std::__transform(__policy, std::move(__first), std::move(__last), std::move(__result), std::move(__op)); if (!__res) std::__throw_bad_alloc(); @@ -100,10 +101,11 @@ _LIBCPP_HIDE_FROM_ABI _ForwardOutIterator transform( _ForwardIterator2 __first2, _ForwardOutIterator __result, _BinaryOperation __op) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1); - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2); - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardOutIterator); - _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(_ForwardOutIterator, decltype(__op(*__first1, *__first2))); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1, "transform requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2, "transform requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardOutIterator, "transform requires an OutputIterator"); + _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR( + _ForwardOutIterator, decltype(__op(*__first1, *__first2)), "transform requires an OutputIterator"); auto __res = std::__transform( __policy, std::move(__first1), std::move(__last1), std::move(__first2), std::move(__result), std::move(__op)); if (!__res) diff --git a/libcxx/include/__iterator/cpp17_iterator_concepts.h b/libcxx/include/__iterator/cpp17_iterator_concepts.h index cdb561e68452..9d5a392582da 100644 --- a/libcxx/include/__iterator/cpp17_iterator_concepts.h +++ b/libcxx/include/__iterator/cpp17_iterator_concepts.h @@ -157,29 +157,31 @@ concept __cpp17_random_access_iterator = _LIBCPP_END_NAMESPACE_STD # ifndef _LIBCPP_DISABLE_ITERATOR_CHECKS -# define _LIBCPP_REQUIRE_CPP17_INPUT_ITERATOR(iter_t) static_assert(::std::__cpp17_input_iterator); -# define _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(iter_t, write_t) \ - static_assert(::std::__cpp17_output_iterator); -# define _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(iter_t) static_assert(::std::__cpp17_forward_iterator); -# define _LIBCPP_REQUIRE_CPP17_BIDIRECTIONAL_ITERATOR(iter_t) \ - static_assert(::std::__cpp17_bidirectional_iterator); -# define _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(iter_t) \ - static_assert(::std::__cpp17_random_access_iterator); +# define _LIBCPP_REQUIRE_CPP17_INPUT_ITERATOR(iter_t, message) \ + static_assert(::std::__cpp17_input_iterator, message) +# define _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(iter_t, write_t, message) \ + static_assert(::std::__cpp17_output_iterator, message) +# define _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(iter_t, message) \ + static_assert(::std::__cpp17_forward_iterator, message) +# define _LIBCPP_REQUIRE_CPP17_BIDIRECTIONAL_ITERATOR(iter_t, message) \ + static_assert(::std::__cpp17_bidirectional_iterator, message) +# define _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(iter_t, message) \ + static_assert(::std::__cpp17_random_access_iterator, message) # else -# define _LIBCPP_REQUIRE_CPP17_INPUT_ITERATOR(iter_t) -# define _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(iter_t, write_t) -# define _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(iter_t) -# define _LIBCPP_REQUIRE_CPP17_BIDIRECTIONAL_ITERATOR(iter_t) -# define _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(iter_t) +# define _LIBCPP_REQUIRE_CPP17_INPUT_ITERATOR(iter_t, message) static_assert(true) +# define _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(iter_t, write_t, message) static_assert(true) +# define _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(iter_t, message) static_assert(true) +# define _LIBCPP_REQUIRE_CPP17_BIDIRECTIONAL_ITERATOR(iter_t, message) static_assert(true) +# define _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(iter_t, message) static_assert(true) # endif #else // _LIBCPP_STD_VER >= 20 -# define _LIBCPP_REQUIRE_CPP17_INPUT_ITERATOR(iter_t) -# define _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(iter_t, write_t) -# define _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(iter_t) -# define _LIBCPP_REQUIRE_CPP17_BIDIRECTIONAL_ITERATOR(iter_t) -# define _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(iter_t) +# define _LIBCPP_REQUIRE_CPP17_INPUT_ITERATOR(iter_t, message) static_assert(true) +# define _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(iter_t, write_t, message) static_assert(true) +# define _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(iter_t, message) static_assert(true) +# define _LIBCPP_REQUIRE_CPP17_BIDIRECTIONAL_ITERATOR(iter_t, message) static_assert(true) +# define _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(iter_t, message) static_assert(true) #endif // _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__numeric/pstl_reduce.h b/libcxx/include/__numeric/pstl_reduce.h index f9f666c2bb38..d678b9480070 100644 --- a/libcxx/include/__numeric/pstl_reduce.h +++ b/libcxx/include/__numeric/pstl_reduce.h @@ -12,6 +12,7 @@ #include <__algorithm/pstl_frontend_dispatch.h> #include <__config> #include <__functional/identity.h> +#include <__iterator/cpp17_iterator_concepts.h> #include <__iterator/iterator_traits.h> #include <__numeric/pstl_transform_reduce.h> #include <__type_traits/is_execution_policy.h> @@ -66,6 +67,7 @@ reduce(_ExecutionPolicy&& __policy, _ForwardIterator __last, _Tp __init, _BinaryOperation __op = {}) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "reduce requires ForwardIterators"); auto __res = std::__reduce(__policy, std::move(__first), std::move(__last), std::move(__init), std::move(__op)); if (!__res) std::__throw_bad_alloc(); @@ -94,6 +96,7 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI __iter_value_type<_ForwardIterator> reduce(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "reduce requires ForwardIterators"); auto __res = std::__reduce(__policy, std::move(__first), std::move(__last)); if (!__res) std::__throw_bad_alloc(); diff --git a/libcxx/include/__numeric/pstl_transform_reduce.h b/libcxx/include/__numeric/pstl_transform_reduce.h index 07ecf0d9956b..2d2621dc8dad 100644 --- a/libcxx/include/__numeric/pstl_transform_reduce.h +++ b/libcxx/include/__numeric/pstl_transform_reduce.h @@ -13,6 +13,7 @@ #include <__algorithm/pstl_frontend_dispatch.h> #include <__config> #include <__functional/operations.h> +#include <__iterator/cpp17_iterator_concepts.h> #include <__numeric/transform_reduce.h> #include <__type_traits/is_execution_policy.h> #include <__utility/move.h> @@ -72,6 +73,8 @@ _LIBCPP_HIDE_FROM_ABI _Tp transform_reduce( _Tp __init, _BinaryOperation1 __reduce, _BinaryOperation2 __transform) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1, "transform_reduce requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2, "transform_reduce requires ForwardIterators"); auto __res = std::__transform_reduce( __policy, std::move(__first1), @@ -99,6 +102,8 @@ _LIBCPP_HIDE_FROM_ABI _Tp transform_reduce( _ForwardIterator1 __last1, _ForwardIterator2 __first2, _Tp __init) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1, "transform_reduce requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2, "transform_reduce requires ForwardIterators"); return std::transform_reduce(__policy, __first1, __last1, __first2, __init, plus{}, multiplies{}); } @@ -140,6 +145,7 @@ _LIBCPP_HIDE_FROM_ABI _Tp transform_reduce( _Tp __init, _BinaryOperation __reduce, _UnaryOperation __transform) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "transform_reduce requires ForwardIterators"); auto __res = std::__transform_reduce( __policy, std::move(__first), std::move(__last), std::move(__init), std::move(__reduce), std::move(__transform)); if (!__res) diff --git a/libcxx/test/libcxx/algorithms/cpp17_iterator_concepts.verify.cpp b/libcxx/test/libcxx/algorithms/cpp17_iterator_concepts.verify.cpp index 344543d5f19f..544a9744b790 100644 --- a/libcxx/test/libcxx/algorithms/cpp17_iterator_concepts.verify.cpp +++ b/libcxx/test/libcxx/algorithms/cpp17_iterator_concepts.verify.cpp @@ -16,29 +16,29 @@ #include struct missing_deref { - using difference_type = std::ptrdiff_t; + using difference_type = std::ptrdiff_t; using iterator_category = std::input_iterator_tag; - using value_type = int; - using reference = int&; + using value_type = int; + using reference = int&; missing_deref& operator++(); }; struct missing_preincrement { - using difference_type = std::ptrdiff_t; + using difference_type = std::ptrdiff_t; using iterator_category = std::input_iterator_tag; - using value_type = int; - using reference = int&; + using value_type = int; + using reference = int&; int& operator*(); }; template struct valid_iterator { - using difference_type = std::ptrdiff_t; + using difference_type = std::ptrdiff_t; using iterator_category = std::input_iterator_tag; - using value_type = int; - using reference = int&; + using value_type = int; + using reference = int&; int& operator*() const; Derived& operator++(); @@ -51,30 +51,30 @@ struct valid_iterator { }; struct not_move_constructible : valid_iterator { - not_move_constructible(const not_move_constructible&) = default; - not_move_constructible(not_move_constructible&&) = delete; - not_move_constructible& operator=(not_move_constructible&&) = default; + not_move_constructible(const not_move_constructible&) = default; + not_move_constructible(not_move_constructible&&) = delete; + not_move_constructible& operator=(not_move_constructible&&) = default; not_move_constructible& operator=(const not_move_constructible&) = default; }; struct not_copy_constructible : valid_iterator { - not_copy_constructible(const not_copy_constructible&) = delete; - not_copy_constructible(not_copy_constructible&&) = default; - not_copy_constructible& operator=(not_copy_constructible&&) = default; + not_copy_constructible(const not_copy_constructible&) = delete; + not_copy_constructible(not_copy_constructible&&) = default; + not_copy_constructible& operator=(not_copy_constructible&&) = default; not_copy_constructible& operator=(const not_copy_constructible&) = default; }; struct not_move_assignable : valid_iterator { - not_move_assignable(const not_move_assignable&) = default; - not_move_assignable(not_move_assignable&&) = default; - not_move_assignable& operator=(not_move_assignable&&) = delete; + not_move_assignable(const not_move_assignable&) = default; + not_move_assignable(not_move_assignable&&) = default; + not_move_assignable& operator=(not_move_assignable&&) = delete; not_move_assignable& operator=(const not_move_assignable&) = default; }; struct not_copy_assignable : valid_iterator { - not_copy_assignable(const not_copy_assignable&) = default; - not_copy_assignable(not_copy_assignable&&) = default; - not_copy_assignable& operator=(not_copy_assignable&&) = default; + not_copy_assignable(const not_copy_assignable&) = default; + not_copy_assignable(not_copy_assignable&&) = default; + not_copy_assignable& operator=(not_copy_assignable&&) = default; not_copy_assignable& operator=(const not_copy_assignable&) = delete; }; @@ -89,7 +89,6 @@ void check_iterator_requirements() { static_assert(std::__cpp17_iterator); // expected-error {{static assertion failed}} // expected-note@*:* {{cannot increment value of type 'missing_preincrement'}} - static_assert(std::__cpp17_iterator); // expected-error {{static assertion failed}} // expected-note@*:* {{because 'not_move_constructible' does not satisfy '__cpp17_move_constructible'}} @@ -115,11 +114,13 @@ bool operator==(not_unequality_comparable, not_unequality_comparable); bool operator!=(not_unequality_comparable, not_unequality_comparable) = delete; void check_input_iterator_requirements() { - _LIBCPP_REQUIRE_CPP17_INPUT_ITERATOR(not_equality_comparable); // expected-error {{static assertion failed}} + // clang-format off + _LIBCPP_REQUIRE_CPP17_INPUT_ITERATOR(not_equality_comparable, ""); // expected-error {{static assertion failed}} // expected-note@*:* {{'__lhs == __rhs' would be invalid: overload resolution selected deleted operator '=='}} - _LIBCPP_REQUIRE_CPP17_INPUT_ITERATOR(not_unequality_comparable); // expected-error {{static assertion failed}} + _LIBCPP_REQUIRE_CPP17_INPUT_ITERATOR(not_unequality_comparable, ""); // expected-error {{static assertion failed}} // expected-note@*:* {{'__lhs != __rhs' would be invalid: overload resolution selected deleted operator '!='}} + // clang-format on } template @@ -138,9 +139,9 @@ struct postincrement_not_ref : valid_iterator {}; bool operator==(postincrement_not_ref, postincrement_not_ref); void check_forward_iterator_requirements() { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(not_default_constructible); // expected-error {{static assertion failed}} + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(not_default_constructible, ""); // expected-error {{static assertion failed}} // expected-note@*:* {{because 'not_default_constructible' does not satisfy '__cpp17_default_constructible'}} - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(postincrement_not_ref); // expected-error {{static assertion failed}} + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(postincrement_not_ref, ""); // expected-error {{static assertion failed}} #ifndef _AIX // expected-note@*:* {{because type constraint 'convertible_to::Proxy, const postincrement_not_ref &>' was not satisfied}} #endif @@ -155,7 +156,6 @@ struct missing_postdecrement : valid_forward_iterator { }; struct not_returning_iter_reference : valid_forward_iterator { - struct Proxy { operator const not_returning_iter_reference&(); @@ -167,12 +167,14 @@ struct not_returning_iter_reference : valid_forward_iterator >' was not satisfied}} + // clang-format on } template @@ -246,7 +248,8 @@ struct missing_minus_const_iter_const_iter : valid_random_access_iterator { @@ -359,62 +362,64 @@ struct missing_const_const_greater_eq : valid_random_access_iterator __iter' would be invalid: overload resolution selected deleted operator '>'}} - _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_const_mut_greater); // expected-error {{static assertion failed}} + _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_const_mut_greater, ""); // expected-error {{static assertion failed}} // expected-note@*:* {{because 'std::as_const(__iter) > __iter' would be invalid: overload resolution selected deleted operator '>'}} - _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_mut_const_greater); // expected-error {{static assertion failed}} + _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_mut_const_greater, ""); // expected-error {{static assertion failed}} // expected-note@*:* {{because '__iter > std::as_const(__iter)' would be invalid: overload resolution selected deleted operator '>'}} - _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_const_const_greater); // expected-error {{static assertion failed}} + _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_const_const_greater, ""); // expected-error {{static assertion failed}} // expected-note@*:* {{because 'std::as_const(__iter) > std::as_const(__iter)' would be invalid: overload resolution selected deleted operator '>'}} - _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_less_eq); // expected-error {{static assertion failed}} + _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_less_eq, ""); // expected-error {{static assertion failed}} // expected-note@*:* {{because '__iter <= __iter' would be invalid: overload resolution selected deleted operator '<='}} - _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_const_mut_less_eq); // expected-error {{static assertion failed}} + _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_const_mut_less_eq, ""); // expected-error {{static assertion failed}} // expected-note@*:* {{because 'std::as_const(__iter) <= __iter' would be invalid: overload resolution selected deleted operator '<='}} - _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_mut_const_less_eq); // expected-error {{static assertion failed}} + _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_mut_const_less_eq, ""); // expected-error {{static assertion failed}} // expected-note@*:* {{because '__iter <= std::as_const(__iter)' would be invalid: overload resolution selected deleted operator '<='}} - _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_const_const_less_eq); // expected-error {{static assertion failed}} + _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_const_const_less_eq, ""); // expected-error {{static assertion failed}} // expected-note@*:* {{because 'std::as_const(__iter) <= std::as_const(__iter)' would be invalid: overload resolution selected deleted operator '<='}} - _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_greater_eq); // expected-error {{static assertion failed}} + _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_greater_eq, ""); // expected-error {{static assertion failed}} // expected-note@*:* {{because '__iter >= __iter' would be invalid: overload resolution selected deleted operator '>='}} - _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_const_mut_greater_eq); // expected-error {{static assertion failed}} + _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_const_mut_greater_eq, ""); // expected-error {{static assertion failed}} // expected-note@*:* {{because 'std::as_const(__iter) >= __iter' would be invalid: overload resolution selected deleted operator '>='}} - _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_mut_const_greater_eq); // expected-error {{static assertion failed}} + _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_mut_const_greater_eq, ""); // expected-error {{static assertion failed}} // expected-note@*:* {{because '__iter >= std::as_const(__iter)' would be invalid: overload resolution selected deleted operator '>='}} - _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_const_const_greater_eq); // expected-error {{static assertion failed}} + _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_const_const_greater_eq, ""); // expected-error {{static assertion failed}} // expected-note@*:* {{because 'std::as_const(__iter) >= std::as_const(__iter)' would be invalid: overload resolution selected deleted operator '>='}} + // clang-format on } diff --git a/libcxx/test/libcxx/algorithms/pstl.iterator-requirements.verify.cpp b/libcxx/test/libcxx/algorithms/pstl.iterator-requirements.verify.cpp new file mode 100644 index 000000000000..98e3509752e1 --- /dev/null +++ b/libcxx/test/libcxx/algorithms/pstl.iterator-requirements.verify.cpp @@ -0,0 +1,192 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14 +// REQUIRES: stdlib=libc++ +// UNSUPPORTED: libcpp-has-no-incomplete-pstl + +// +// + +// Make sure that all PSTL algorithms contain checks for iterator requirements. +// This is not a requirement from the Standard, but we strive to catch misuse in +// the PSTL both because we can, and because iterator category mistakes in the +// PSTL can lead to subtle bugs. + +// Ignore spurious errors after the initial static_assert failure. +// ADDITIONAL_COMPILE_FLAGS: -Xclang -verify-ignore-unexpected=error + +// We only diagnose this in C++20 and above because we implement the checks with concepts. +// UNSUPPORTED: c++17 + +#include +#include +#include + +#include "test_iterators.h" + +using non_forward_iterator = cpp17_input_iterator; +struct non_output_iterator : forward_iterator { + constexpr int const& operator*() const; // prevent it from being an output iterator +}; + +void f(non_forward_iterator non_fwd, non_output_iterator non_output, std::execution::sequenced_policy pol) { + auto pred = [](auto&&...) -> bool { return true; }; + auto func = [](auto&&...) -> int { return 1; }; + int* it = nullptr; + int* out = nullptr; + std::size_t n = 0; + int val = 0; + + { + (void)std::any_of(pol, non_fwd, non_fwd, pred); // expected-error@*:* {{static assertion failed: any_of}} + (void)std::all_of(pol, non_fwd, non_fwd, pred); // expected-error@*:* {{static assertion failed: all_of}} + (void)std::none_of(pol, non_fwd, non_fwd, pred); // expected-error@*:* {{static assertion failed: none_of}} + } + + { + (void)std::copy(pol, non_fwd, non_fwd, it); // expected-error@*:* {{static assertion failed: copy}} + (void)std::copy(pol, it, it, non_fwd); // expected-error@*:* {{static assertion failed: copy}} + (void)std::copy(pol, it, it, non_output); // expected-error@*:* {{static assertion failed: copy}} + } + { + (void)std::copy_n(pol, non_fwd, n, it); // expected-error@*:* {{static assertion failed: copy_n}} + (void)std::copy_n(pol, it, n, non_fwd); // expected-error@*:* {{static assertion failed: copy_n}} + (void)std::copy_n(pol, it, n, non_output); // expected-error@*:* {{static assertion failed: copy_n}} + } + + { + (void)std::count(pol, non_fwd, non_fwd, val); // expected-error@*:* {{static assertion failed: count}} + (void)std::count_if(pol, non_fwd, non_fwd, pred); // expected-error@*:* {{static assertion failed: count_if}} + } + + { + (void)std::equal(pol, non_fwd, non_fwd, it); // expected-error@*:* {{static assertion failed: equal}} + (void)std::equal(pol, it, it, non_fwd); // expected-error@*:* {{static assertion failed: equal}} + (void)std::equal(pol, non_fwd, non_fwd, it, pred); // expected-error@*:* {{static assertion failed: equal}} + (void)std::equal(pol, it, it, non_fwd, pred); // expected-error@*:* {{static assertion failed: equal}} + + (void)std::equal(pol, non_fwd, non_fwd, it, it); // expected-error@*:* {{static assertion failed: equal}} + (void)std::equal(pol, it, it, non_fwd, non_fwd); // expected-error@*:* {{static assertion failed: equal}} + (void)std::equal(pol, non_fwd, non_fwd, it, it, pred); // expected-error@*:* {{static assertion failed: equal}} + (void)std::equal(pol, it, it, non_fwd, non_fwd, pred); // expected-error@*:* {{static assertion failed: equal}} + } + + { + (void)std::fill(pol, non_fwd, non_fwd, val); // expected-error@*:* {{static assertion failed: fill}} + (void)std::fill_n(pol, non_fwd, n, val); // expected-error@*:* {{static assertion failed: fill_n}} + } + + { + (void)std::find(pol, non_fwd, non_fwd, val); // expected-error@*:* {{static assertion failed: find}} + (void)std::find_if(pol, non_fwd, non_fwd, pred); // expected-error@*:* {{static assertion failed: find_if}} + (void)std::find_if_not(pol, non_fwd, non_fwd, pred); // expected-error@*:* {{static assertion failed: find_if_not}} + } + + { + (void)std::for_each(pol, non_fwd, non_fwd, func); // expected-error@*:* {{static assertion failed: for_each}} + (void)std::for_each_n(pol, non_fwd, n, func); // expected-error@*:* {{static assertion failed: for_each_n}} + } + + { + (void)std::generate(pol, non_fwd, non_fwd, func); // expected-error@*:* {{static assertion failed: generate}} + (void)std::generate_n(pol, non_fwd, n, func); // expected-error@*:* {{static assertion failed: generate_n}} + } + + { + (void)std::is_partitioned( + pol, non_fwd, non_fwd, pred); // expected-error@*:* {{static assertion failed: is_partitioned}} + } + + { + (void)std::merge(pol, non_fwd, non_fwd, it, it, out); // expected-error@*:* {{static assertion failed: merge}} + (void)std::merge(pol, it, it, non_fwd, non_fwd, out); // expected-error@*:* {{static assertion failed: merge}} + (void)std::merge(pol, it, it, it, it, non_output); // expected-error@*:* {{static assertion failed: merge}} + + (void)std::merge(pol, non_fwd, non_fwd, it, it, out, pred); // expected-error@*:* {{static assertion failed: merge}} + (void)std::merge(pol, it, it, non_fwd, non_fwd, out, pred); // expected-error@*:* {{static assertion failed: merge}} + (void)std::merge(pol, it, it, it, it, non_output, pred); // expected-error@*:* {{static assertion failed: merge}} + } + + { + (void)std::move(pol, non_fwd, non_fwd, out); // expected-error@*:* {{static assertion failed: move}} + (void)std::move(pol, it, it, non_fwd); // expected-error@*:* {{static assertion failed: move}} + (void)std::move(pol, it, it, non_output); // expected-error@*:* {{static assertion failed: move}} + } + + { + (void)std::replace_if( + pol, non_fwd, non_fwd, pred, val); // expected-error@*:* {{static assertion failed: replace_if}} + (void)std::replace(pol, non_fwd, non_fwd, val, val); // expected-error@*:* {{static assertion failed: replace}} + + (void)std::replace_copy_if( + pol, non_fwd, non_fwd, out, pred, val); // expected-error@*:* {{static assertion failed: replace_copy_if}} + (void)std::replace_copy_if( + pol, it, it, non_fwd, pred, val); // expected-error@*:* {{static assertion failed: replace_copy_if}} + (void)std::replace_copy_if( + pol, it, it, non_output, pred, val); // expected-error@*:* {{static assertion failed: replace_copy_if}} + + (void)std::replace_copy( + pol, non_fwd, non_fwd, out, val, val); // expected-error@*:* {{static assertion failed: replace_copy}} + (void)std::replace_copy( + pol, it, it, non_fwd, val, val); // expected-error@*:* {{static assertion failed: replace_copy}} + (void)std::replace_copy( + pol, it, it, non_output, val, val); // expected-error@*:* {{static assertion failed: replace_copy}} + } + + { + (void)std::rotate_copy( + pol, non_fwd, non_fwd, non_fwd, out); // expected-error@*:* {{static assertion failed: rotate_copy}} + (void)std::rotate_copy(pol, it, it, it, non_fwd); // expected-error@*:* {{static assertion failed: rotate_copy}} + (void)std::rotate_copy(pol, it, it, it, non_output); // expected-error@*:* {{static assertion failed: rotate_copy}} + } + + { + (void)std::sort(pol, non_fwd, non_fwd); // expected-error@*:* {{static assertion failed: sort}} + (void)std::sort(pol, non_fwd, non_fwd, pred); // expected-error@*:* {{static assertion failed: sort}} + } + + { + (void)std::stable_sort(pol, non_fwd, non_fwd); // expected-error@*:* {{static assertion failed: stable_sort}} + (void)std::stable_sort(pol, non_fwd, non_fwd, pred); // expected-error@*:* {{static assertion failed: stable_sort}} + } + + { + (void)std::transform(pol, non_fwd, non_fwd, out, func); // expected-error@*:* {{static assertion failed: transform}} + (void)std::transform(pol, it, it, non_fwd, func); // expected-error@*:* {{static assertion failed: transform}} + (void)std::transform(pol, it, it, non_output, func); // expected-error@*:* {{static assertion failed: transform}} + + (void)std::transform( + pol, non_fwd, non_fwd, it, out, func); // expected-error@*:* {{static assertion failed: transform}} + (void)std::transform(pol, it, it, non_fwd, out, func); // expected-error@*:* {{static assertion failed: transform}} + (void)std::transform(pol, it, it, it, non_fwd, func); // expected-error@*:* {{static assertion failed: transform}} + (void)std::transform( + pol, it, it, it, non_output, func); // expected-error@*:* {{static assertion failed: transform}} + } + + { + (void)std::reduce(pol, non_fwd, non_fwd); // expected-error@*:* {{static assertion failed: reduce}} + (void)std::reduce(pol, non_fwd, non_fwd, val); // expected-error@*:* {{static assertion failed: reduce}} + (void)std::reduce(pol, non_fwd, non_fwd, val, func); // expected-error@*:* {{static assertion failed: reduce}} + } + + { + (void)std::transform_reduce( + pol, non_fwd, non_fwd, it, val); // expected-error@*:* {{static assertion failed: transform_reduce}} + (void)std::transform_reduce( + pol, it, it, non_fwd, val); // expected-error@*:* {{static assertion failed: transform_reduce}} + + (void)std::transform_reduce( + pol, non_fwd, non_fwd, it, val, func, func); // expected-error@*:* {{static assertion failed: transform_reduce}} + (void)std::transform_reduce( + pol, it, it, non_fwd, val, func, func); // expected-error@*:* {{static assertion failed: transform_reduce}} + + (void)std::transform_reduce( + pol, non_fwd, non_fwd, val, func, func); // expected-error@*:* {{static assertion failed: transform_reduce}} + } +} diff --git a/libcxx/test/support/test_iterators.h b/libcxx/test/support/test_iterators.h index 7ffb74990fa4..aa819ecd4733 100644 --- a/libcxx/test/support/test_iterators.h +++ b/libcxx/test/support/test_iterators.h @@ -1484,9 +1484,14 @@ public: return tmp; } - iterator_wrapper& operator+=(difference_type i) { + Derived& operator+=(difference_type i) { iter_ += i; - return *this; + return static_cast(*this); + } + + Derived& operator-=(difference_type i) { + iter_ -= i; + return static_cast(*this); } friend decltype(iter_ - iter_) operator-(const iterator_wrapper& lhs, const iterator_wrapper& rhs) { @@ -1503,8 +1508,15 @@ public: return iter; } + friend Derived operator+(difference_type i, Derived iter) { return iter + i; } + friend bool operator==(const iterator_wrapper& lhs, const iterator_wrapper& rhs) { return lhs.iter_ == rhs.iter_; } friend bool operator!=(const iterator_wrapper& lhs, const iterator_wrapper& rhs) { return lhs.iter_ != rhs.iter_; } + + friend bool operator>(const iterator_wrapper& lhs, const iterator_wrapper& rhs) { return lhs.iter_ > rhs.iter_; } + friend bool operator<(const iterator_wrapper& lhs, const iterator_wrapper& rhs) { return lhs.iter_ < rhs.iter_; } + friend bool operator<=(const iterator_wrapper& lhs, const iterator_wrapper& rhs) { return lhs.iter_ <= rhs.iter_; } + friend bool operator>=(const iterator_wrapper& lhs, const iterator_wrapper& rhs) { return lhs.iter_ >= rhs.iter_; } }; class iterator_error : std::runtime_error { -- GitLab From 6c7853080451a7f9ba612f57b41a076061b2a1a9 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Wed, 17 Apr 2024 13:18:55 +0100 Subject: [PATCH 040/862] [X86] vector-shuffle-combining-sse41.ll - add missing AVX1/2/512 check prefixes --- .../X86/vector-shuffle-combining-sse41.ll | 133 +++++++++++++++++- 1 file changed, 129 insertions(+), 4 deletions(-) diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining-sse41.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining-sse41.ll index 33851f56fe8d..8d213d257743 100644 --- a/llvm/test/CodeGen/X86/vector-shuffle-combining-sse41.ll +++ b/llvm/test/CodeGen/X86/vector-shuffle-combining-sse41.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE -; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx | FileCheck %s --check-prefix=AVX -; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX -; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX +; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE +; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1 +; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2 +; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX,AVX512 ; Combine tests involving SSE41 target shuffles (BLEND,INSERTPS,MOVZX) @@ -29,6 +29,30 @@ define <4 x i32> @combine_blend_of_permutes_v4i32(<2 x i64> %a0, <2 x i64> %a1) ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1] ; SSE-NEXT: pblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3],xmm2[4,5],xmm0[6,7] ; SSE-NEXT: retq +; +; AVX1-LABEL: combine_blend_of_permutes_v4i32: +; AVX1: # %bb.0: +; AVX1-NEXT: vshufps {{.*#+}} xmm0 = xmm0[2,3,0,1] +; AVX1-NEXT: vshufps {{.*#+}} xmm1 = xmm1[2,3,0,1] +; AVX1-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3] +; AVX1-NEXT: retq +; +; AVX2-LABEL: combine_blend_of_permutes_v4i32: +; AVX2: # %bb.0: +; AVX2-NEXT: vshufps {{.*#+}} xmm0 = xmm0[2,3,0,1] +; AVX2-NEXT: vshufps {{.*#+}} xmm1 = xmm1[2,3,0,1] +; AVX2-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3] +; AVX2-NEXT: retq +; +; AVX512-LABEL: combine_blend_of_permutes_v4i32: +; AVX512: # %bb.0: +; AVX512-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1 +; AVX512-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 +; AVX512-NEXT: vpmovsxbd {{.*#+}} xmm2 = [2,19,0,17] +; AVX512-NEXT: vpermt2d %zmm1, %zmm2, %zmm0 +; AVX512-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 +; AVX512-NEXT: vzeroupper +; AVX512-NEXT: retq %s0 = shufflevector <2 x i64> %a0, <2 x i64> undef, <2 x i32> %s1 = shufflevector <2 x i64> %a1, <2 x i64> undef, <2 x i32> %x0 = bitcast <2 x i64> %s0 to <4 x i32> @@ -71,6 +95,107 @@ define <16 x i8> @PR50049(ptr %p1, ptr %p2) { ; SSE-NEXT: pand %xmm5, %xmm1 ; SSE-NEXT: packuswb %xmm1, %xmm0 ; SSE-NEXT: retq +; +; AVX1-LABEL: PR50049: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 +; AVX1-NEXT: vmovdqa 32(%rdi), %xmm2 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [1,4,7,10,13,128,128,128,128,128,128,u,u,u,u,u] +; AVX1-NEXT: vpshufb %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [128,128,128,128,128,0,3,6,9,12,15,u,u,u,u,u] +; AVX1-NEXT: vpshufb %xmm4, %xmm0, %xmm0 +; AVX1-NEXT: vpor %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa (%rsi), %xmm2 +; AVX1-NEXT: vmovdqa 16(%rsi), %xmm5 +; AVX1-NEXT: vmovdqa 32(%rsi), %xmm6 +; AVX1-NEXT: vpshufb %xmm3, %xmm6, %xmm3 +; AVX1-NEXT: vpshufb %xmm4, %xmm2, %xmm2 +; AVX1-NEXT: vpor %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [5,6,7,8,9,10,128,128,128,128,128,0,1,2,3,4] +; AVX1-NEXT: vpshufb %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [128,128,128,128,128,128,2,5,8,11,14,128,128,128,128,128] +; AVX1-NEXT: vpshufb %xmm4, %xmm5, %xmm5 +; AVX1-NEXT: vpor %xmm5, %xmm2, %xmm2 +; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm5 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15] +; AVX1-NEXT: vpshufb %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpshufb %xmm4, %xmm1, %xmm1 +; AVX1-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm1 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15] +; AVX1-NEXT: vpmullw %xmm5, %xmm1, %xmm1 +; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255] +; AVX1-NEXT: vpand %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero +; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero +; AVX1-NEXT: vpmullw %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: PR50049: +; AVX2: # %bb.0: +; AVX2-NEXT: vmovdqa (%rdi), %xmm0 +; AVX2-NEXT: vmovdqa 16(%rdi), %xmm1 +; AVX2-NEXT: vmovdqa 32(%rdi), %xmm2 +; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [1,4,7,10,13,128,128,128,128,128,128,u,u,u,u,u] +; AVX2-NEXT: vpshufb %xmm3, %xmm2, %xmm2 +; AVX2-NEXT: vmovdqa {{.*#+}} xmm4 = [128,128,128,128,128,0,3,6,9,12,15,u,u,u,u,u] +; AVX2-NEXT: vpshufb %xmm4, %xmm0, %xmm0 +; AVX2-NEXT: vpor %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vmovdqa (%rsi), %xmm2 +; AVX2-NEXT: vmovdqa 16(%rsi), %xmm5 +; AVX2-NEXT: vmovdqa 32(%rsi), %xmm6 +; AVX2-NEXT: vpshufb %xmm3, %xmm6, %xmm3 +; AVX2-NEXT: vpshufb %xmm4, %xmm2, %xmm2 +; AVX2-NEXT: vpor %xmm3, %xmm2, %xmm2 +; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [5,6,7,8,9,10,128,128,128,128,128,0,1,2,3,4] +; AVX2-NEXT: vpshufb %xmm3, %xmm2, %xmm2 +; AVX2-NEXT: vmovdqa {{.*#+}} xmm4 = [128,128,128,128,128,128,2,5,8,11,14,128,128,128,128,128] +; AVX2-NEXT: vpshufb %xmm4, %xmm5, %xmm5 +; AVX2-NEXT: vpor %xmm5, %xmm2, %xmm2 +; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero +; AVX2-NEXT: vpshufb %xmm3, %xmm0, %xmm0 +; AVX2-NEXT: vpshufb %xmm4, %xmm1, %xmm1 +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero +; AVX2-NEXT: vpmullw %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 +; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 +; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq +; +; AVX512-LABEL: PR50049: +; AVX512: # %bb.0: +; AVX512-NEXT: vmovdqa (%rdi), %xmm0 +; AVX512-NEXT: vmovdqa 16(%rdi), %xmm1 +; AVX512-NEXT: vmovdqa 32(%rdi), %xmm2 +; AVX512-NEXT: vmovdqa {{.*#+}} xmm3 = [1,4,7,10,13,128,128,128,128,128,128,u,u,u,u,u] +; AVX512-NEXT: vpshufb %xmm3, %xmm2, %xmm2 +; AVX512-NEXT: vmovdqa {{.*#+}} xmm4 = [128,128,128,128,128,0,3,6,9,12,15,u,u,u,u,u] +; AVX512-NEXT: vpshufb %xmm4, %xmm0, %xmm0 +; AVX512-NEXT: vpor %xmm2, %xmm0, %xmm0 +; AVX512-NEXT: vmovdqa (%rsi), %xmm2 +; AVX512-NEXT: vmovdqa 16(%rsi), %xmm5 +; AVX512-NEXT: vmovdqa 32(%rsi), %xmm6 +; AVX512-NEXT: vpshufb %xmm3, %xmm6, %xmm3 +; AVX512-NEXT: vpshufb %xmm4, %xmm2, %xmm2 +; AVX512-NEXT: vpor %xmm3, %xmm2, %xmm2 +; AVX512-NEXT: vmovdqa {{.*#+}} xmm3 = [5,6,7,8,9,10,128,128,128,128,128,0,1,2,3,4] +; AVX512-NEXT: vpshufb %xmm3, %xmm2, %xmm2 +; AVX512-NEXT: vmovdqa {{.*#+}} xmm4 = [128,128,128,128,128,128,2,5,8,11,14,128,128,128,128,128] +; AVX512-NEXT: vpshufb %xmm4, %xmm5, %xmm5 +; AVX512-NEXT: vpor %xmm5, %xmm2, %xmm2 +; AVX512-NEXT: vpmovzxbw {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero +; AVX512-NEXT: vpshufb %xmm3, %xmm0, %xmm0 +; AVX512-NEXT: vpshufb %xmm4, %xmm1, %xmm1 +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero +; AVX512-NEXT: vpmullw %ymm2, %ymm0, %ymm0 +; AVX512-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero +; AVX512-NEXT: vpmovdb %zmm0, %xmm0 +; AVX512-NEXT: vzeroupper +; AVX512-NEXT: retq %x1 = load <48 x i8>, ptr %p1, align 16 %x2 = load <48 x i8>, ptr %p2, align 16 %s1 = shufflevector <48 x i8> %x1, <48 x i8> poison, <16 x i32> -- GitLab From 37b26bf48b9894ed0c13fd1aede23472660fb75e Mon Sep 17 00:00:00 2001 From: "Oleksandr \"Alex\" Zinenko" Date: Wed, 17 Apr 2024 14:24:51 +0200 Subject: [PATCH 041/862] [mlir] transform.apply_patterns support more config options (#88484) Greedy rewrite driver has options to control the number of rewrites applies. Expose those via the corresponding transform op. --- .../mlir/Dialect/Transform/IR/TransformOps.td | 5 +++- .../lib/Dialect/Transform/IR/TransformOps.cpp | 7 +++++ .../Transform/test-pattern-application.mlir | 30 +++++++++++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td b/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td index 21c9595860d4..fbac1ffb621f 100644 --- a/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td +++ b/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td @@ -331,7 +331,10 @@ def ApplyPatternsOp : TransformDialectOp<"apply_patterns", }]; let arguments = (ins - TransformHandleTypeInterface:$target, UnitAttr:$apply_cse); + TransformHandleTypeInterface:$target, + UnitAttr:$apply_cse, + DefaultValuedAttr(-1)">:$max_iterations, + DefaultValuedAttr(-1)">:$max_num_rewrites); let results = (outs); let regions = (region MaxSizedRegion<1>:$patterns); diff --git a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp index dc19022219e5..53f958caa0bd 100644 --- a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp +++ b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp @@ -396,6 +396,13 @@ DiagnosedSilenceableFailure transform::ApplyPatternsOp::applyToOne( static_cast(rewriter.getListener()); FrozenRewritePatternSet frozenPatterns(std::move(patterns)); + config.maxIterations = getMaxIterations() == static_cast(-1) + ? GreedyRewriteConfig::kNoLimit + : getMaxIterations(); + config.maxNumRewrites = getMaxNumRewrites() == static_cast(-1) + ? GreedyRewriteConfig::kNoLimit + : getMaxNumRewrites(); + // Apply patterns and CSE repetitively until a fixpoint is reached. If no CSE // was requested, apply the greedy pattern rewrite only once. (The greedy // pattern rewrite driver already iterates to a fixpoint internally.) diff --git a/mlir/test/Dialect/Transform/test-pattern-application.mlir b/mlir/test/Dialect/Transform/test-pattern-application.mlir index fa8a555af921..f78b4b6f6798 100644 --- a/mlir/test/Dialect/Transform/test-pattern-application.mlir +++ b/mlir/test/Dialect/Transform/test-pattern-application.mlir @@ -26,6 +26,36 @@ module attributes {transform.with_named_sequence} { // ----- +// CHECK-LABEL: @limited_updates +func.func @limited_updates() { + "test.container"() ({ + // Only one is replaced. + // CHECK: "test.foo"() {replace_with_new_op = "test.foo"} + // CHECK: "test.foo"() : () + %0 = "test.foo"() {replace_with_new_op = "test.foo"} : () -> (i32) + %1 = "test.foo"() {replace_with_new_op = "test.foo"} : () -> (i32) + }) : () -> () + return +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + // Pattern application will fail because of the upper limit, wrap in + // sequence to suppress the error message. + transform.sequence %arg0 : !transform.any_op failures(suppress) { + ^bb0(%arg1: !transform.any_op): + %0 = transform.structured.match ops{["test.container"]} in %arg1 : (!transform.any_op) -> !transform.any_op + %1 = transform.structured.match ops{["test.foo"]} in %arg1 : (!transform.any_op) -> !transform.any_op + transform.apply_patterns to %0 { + transform.apply_patterns.transform.test_patterns + } {max_num_rewrites = 1} : !transform.any_op + } + transform.yield + } +} + +// ----- + func.func @replacement_op_not_found() { "test.container"() ({ // expected-note @below {{[0] replaced op}} -- GitLab From 4536ad47579d8d61f372ab85128bcfaed58a1256 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Wed, 17 Apr 2024 20:24:59 +0800 Subject: [PATCH 042/862] [RISCV] Fix clang-tidy warning about else after return. NFC --- llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp index 6e45f0c703ce..aab91adbb64b 100644 --- a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp +++ b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp @@ -1551,7 +1551,9 @@ void RISCVInsertVSETVLI::doLocalPostpass(MachineBasicBlock &MBB) { ToDelete.push_back(&MI); // Leave NextMI unchanged continue; - } else if (canMutatePriorConfig(MI, *NextMI, Used, *MRI)) { + } + + if (canMutatePriorConfig(MI, *NextMI, Used, *MRI)) { if (!isVLPreservingConfig(*NextMI)) { MI.getOperand(0).setReg(NextMI->getOperand(0).getReg()); MI.getOperand(0).setIsDead(false); -- GitLab From 86a78284e7ce2ecc7a9283c7d141566a32371492 Mon Sep 17 00:00:00 2001 From: Quentin Dian Date: Wed, 17 Apr 2024 20:27:09 +0800 Subject: [PATCH 043/862] [TailDuplicator] Add maximum predecessors and successors to consider tail duplicating blocks (#78582) Fixes #78578. Duplicating a BB which has both multiple predecessors and successors will result in a complex CFG and also may cause huge amount of PHI nodes. See https://github.com/llvm/llvm-project/issues/78578#issuecomment-1962363580 for a detailed description of the limit. --- llvm/lib/CodeGen/TailDuplicator.cpp | 20 ++ .../CodeGen/X86/tail-dup-pred-succ-size.mir | 260 ++++++++++++++++++ 2 files changed, 280 insertions(+) create mode 100644 llvm/test/CodeGen/X86/tail-dup-pred-succ-size.mir diff --git a/llvm/lib/CodeGen/TailDuplicator.cpp b/llvm/lib/CodeGen/TailDuplicator.cpp index 5ed67bd0a121..f5dd21cb9270 100644 --- a/llvm/lib/CodeGen/TailDuplicator.cpp +++ b/llvm/lib/CodeGen/TailDuplicator.cpp @@ -68,6 +68,18 @@ static cl::opt TailDupIndirectBranchSize( "end with indirect branches."), cl::init(20), cl::Hidden); +static cl::opt + TailDupPredSize("tail-dup-pred-size", + cl::desc("Maximum predecessors (maximum successors at the " + "same time) to consider tail duplicating blocks."), + cl::init(16), cl::Hidden); + +static cl::opt + TailDupSuccSize("tail-dup-succ-size", + cl::desc("Maximum successors (maximum predecessors at the " + "same time) to consider tail duplicating blocks."), + cl::init(16), cl::Hidden); + static cl::opt TailDupVerify("tail-dup-verify", cl::desc("Verify sanity of PHI instructions during taildup"), @@ -565,6 +577,14 @@ bool TailDuplicator::shouldTailDuplicate(bool IsSimple, if (TailBB.isSuccessor(&TailBB)) return false; + // Duplicating a BB which has both multiple predecessors and successors will + // result in a complex CFG and also may cause huge amount of PHI nodes. If we + // want to remove this limitation, we have to address + // https://github.com/llvm/llvm-project/issues/78578. + if (TailBB.pred_size() > TailDupPredSize && + TailBB.succ_size() > TailDupSuccSize) + return false; + // Set the limit on the cost to duplicate. When optimizing for size, // duplicate only one, because one branch instruction can be eliminated to // compensate for the duplication. diff --git a/llvm/test/CodeGen/X86/tail-dup-pred-succ-size.mir b/llvm/test/CodeGen/X86/tail-dup-pred-succ-size.mir new file mode 100644 index 000000000000..67f8cc72e0d7 --- /dev/null +++ b/llvm/test/CodeGen/X86/tail-dup-pred-succ-size.mir @@ -0,0 +1,260 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4 +# RUN: llc -mtriple=x86_64-unknown-linux-gnu -run-pass=early-tailduplication -tail-dup-pred-size=3 -tail-dup-succ-size=3 %s -o - | FileCheck %s -check-prefix=LIMIT +# RUN: llc -mtriple=x86_64-unknown-linux-gnu -run-pass=early-tailduplication -tail-dup-pred-size=4 -tail-dup-succ-size=4 %s -o - | FileCheck %s -check-prefix=NOLIMIT + +--- +name: foo +tracksRegLiveness: true +jumpTable: + kind: block-address + entries: + - id: 0 + blocks: [ '%bb.2', '%bb.3', '%bb.4', '%bb.5' ] + - id: 1 + blocks: [ '%bb.9', '%bb.10', '%bb.11', '%bb.12' ] +body: | + ; LIMIT-LABEL: name: foo + ; LIMIT: bb.0: + ; LIMIT-NEXT: successors: %bb.2(0x20000000), %bb.3(0x20000000), %bb.4(0x20000000), %bb.5(0x20000000) + ; LIMIT-NEXT: liveins: $rdi, $esi + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $esi + ; LIMIT-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rdi + ; LIMIT-NEXT: [[SHR32ri:%[0-9]+]]:gr32 = SHR32ri [[COPY]], 1, implicit-def dead $eflags + ; LIMIT-NEXT: [[AND32ri:%[0-9]+]]:gr32 = AND32ri [[SHR32ri]], 7, implicit-def dead $eflags + ; LIMIT-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:gr64_nosp = SUBREG_TO_REG 0, killed [[AND32ri]], %subreg.sub_32bit + ; LIMIT-NEXT: JMP64m $noreg, 8, [[SUBREG_TO_REG]], %jump-table.0, $noreg + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: bb.2: + ; LIMIT-NEXT: successors: %bb.7(0x80000000) + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm [[COPY1]], 1, $noreg, 0, $noreg + ; LIMIT-NEXT: JMP_1 %bb.7 + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: bb.3: + ; LIMIT-NEXT: successors: %bb.7(0x80000000) + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: [[MOV32rm1:%[0-9]+]]:gr32 = MOV32rm [[COPY1]], 1, $noreg, 0, $noreg + ; LIMIT-NEXT: [[SHR32ri1:%[0-9]+]]:gr32 = SHR32ri [[MOV32rm1]], 1, implicit-def dead $eflags + ; LIMIT-NEXT: JMP_1 %bb.7 + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: bb.4: + ; LIMIT-NEXT: successors: %bb.7(0x80000000) + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: [[MOV32rm2:%[0-9]+]]:gr32 = MOV32rm [[COPY1]], 1, $noreg, 0, $noreg + ; LIMIT-NEXT: [[SHR32ri2:%[0-9]+]]:gr32 = SHR32ri [[MOV32rm2]], 2, implicit-def dead $eflags + ; LIMIT-NEXT: JMP_1 %bb.7 + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: bb.5: + ; LIMIT-NEXT: successors: %bb.7(0x80000000) + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: [[MOV32rm3:%[0-9]+]]:gr32 = MOV32rm [[COPY1]], 1, $noreg, 0, $noreg + ; LIMIT-NEXT: [[SHR32ri3:%[0-9]+]]:gr32 = SHR32ri [[MOV32rm3]], 3, implicit-def dead $eflags + ; LIMIT-NEXT: JMP_1 %bb.7 + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: bb.6: + ; LIMIT-NEXT: successors: + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: bb.7: + ; LIMIT-NEXT: successors: %bb.9(0x20000000), %bb.10(0x20000000), %bb.11(0x20000000), %bb.12(0x20000000) + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: [[PHI:%[0-9]+]]:gr32 = PHI [[SHR32ri3]], %bb.5, [[SHR32ri2]], %bb.4, [[SHR32ri1]], %bb.3, [[MOV32rm]], %bb.2 + ; LIMIT-NEXT: [[SHR32ri4:%[0-9]+]]:gr32 = SHR32ri [[COPY]], 2, implicit-def dead $eflags + ; LIMIT-NEXT: [[AND32ri1:%[0-9]+]]:gr32 = AND32ri [[SHR32ri4]], 7, implicit-def dead $eflags + ; LIMIT-NEXT: [[SUBREG_TO_REG1:%[0-9]+]]:gr64_nosp = SUBREG_TO_REG 0, killed [[AND32ri1]], %subreg.sub_32bit + ; LIMIT-NEXT: JMP64m $noreg, 8, [[SUBREG_TO_REG1]], %jump-table.1, $noreg + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: bb.9: + ; LIMIT-NEXT: successors: %bb.13(0x80000000) + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: [[MOV32rm4:%[0-9]+]]:gr32 = MOV32rm [[COPY1]], 1, $noreg, 0, $noreg + ; LIMIT-NEXT: JMP_1 %bb.13 + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: bb.10: + ; LIMIT-NEXT: successors: %bb.13(0x80000000) + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: [[MOV32rm5:%[0-9]+]]:gr32 = MOV32rm [[COPY1]], 1, $noreg, 0, $noreg + ; LIMIT-NEXT: [[SHR32ri5:%[0-9]+]]:gr32 = SHR32ri [[MOV32rm5]], 1, implicit-def dead $eflags + ; LIMIT-NEXT: JMP_1 %bb.13 + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: bb.11: + ; LIMIT-NEXT: successors: %bb.13(0x80000000) + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: [[MOV32rm6:%[0-9]+]]:gr32 = MOV32rm [[COPY1]], 1, $noreg, 0, $noreg + ; LIMIT-NEXT: [[SHR32ri6:%[0-9]+]]:gr32 = SHR32ri [[MOV32rm6]], 2, implicit-def dead $eflags + ; LIMIT-NEXT: JMP_1 %bb.13 + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: bb.12: + ; LIMIT-NEXT: successors: %bb.13(0x80000000) + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: [[MOV32rm7:%[0-9]+]]:gr32 = MOV32rm [[COPY1]], 1, $noreg, 0, $noreg + ; LIMIT-NEXT: [[SHR32ri7:%[0-9]+]]:gr32 = SHR32ri [[MOV32rm7]], 6, implicit-def dead $eflags + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: bb.13: + ; LIMIT-NEXT: [[PHI1:%[0-9]+]]:gr32 = PHI [[SHR32ri7]], %bb.12, [[SHR32ri6]], %bb.11, [[SHR32ri5]], %bb.10, [[MOV32rm4]], %bb.9 + ; LIMIT-NEXT: [[OR32rr:%[0-9]+]]:gr32 = OR32rr [[PHI1]], [[PHI]], implicit-def dead $eflags + ; LIMIT-NEXT: $eax = COPY [[OR32rr]] + ; LIMIT-NEXT: RET 0, $eax + ; + ; NOLIMIT-LABEL: name: foo + ; NOLIMIT: bb.0: + ; NOLIMIT-NEXT: successors: %bb.2(0x20000000), %bb.3(0x20000000), %bb.4(0x20000000), %bb.5(0x20000000) + ; NOLIMIT-NEXT: liveins: $rdi, $esi + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $esi + ; NOLIMIT-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rdi + ; NOLIMIT-NEXT: [[SHR32ri:%[0-9]+]]:gr32 = SHR32ri [[COPY]], 1, implicit-def dead $eflags + ; NOLIMIT-NEXT: [[AND32ri:%[0-9]+]]:gr32 = AND32ri [[SHR32ri]], 7, implicit-def dead $eflags + ; NOLIMIT-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:gr64_nosp = SUBREG_TO_REG 0, killed [[AND32ri]], %subreg.sub_32bit + ; NOLIMIT-NEXT: JMP64m $noreg, 8, [[SUBREG_TO_REG]], %jump-table.0, $noreg + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: bb.2: + ; NOLIMIT-NEXT: successors: %bb.9(0x20000000), %bb.10(0x20000000), %bb.11(0x20000000), %bb.12(0x20000000) + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm [[COPY1]], 1, $noreg, 0, $noreg + ; NOLIMIT-NEXT: [[SHR32ri1:%[0-9]+]]:gr32 = SHR32ri [[COPY]], 2, implicit-def dead $eflags + ; NOLIMIT-NEXT: [[AND32ri1:%[0-9]+]]:gr32 = AND32ri [[SHR32ri1]], 7, implicit-def dead $eflags + ; NOLIMIT-NEXT: [[SUBREG_TO_REG1:%[0-9]+]]:gr64_nosp = SUBREG_TO_REG 0, [[AND32ri1]], %subreg.sub_32bit + ; NOLIMIT-NEXT: JMP64m $noreg, 8, [[SUBREG_TO_REG1]], %jump-table.1, $noreg + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: bb.3: + ; NOLIMIT-NEXT: successors: %bb.9(0x20000000), %bb.10(0x20000000), %bb.11(0x20000000), %bb.12(0x20000000) + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: [[MOV32rm1:%[0-9]+]]:gr32 = MOV32rm [[COPY1]], 1, $noreg, 0, $noreg + ; NOLIMIT-NEXT: [[SHR32ri2:%[0-9]+]]:gr32 = SHR32ri [[MOV32rm1]], 1, implicit-def dead $eflags + ; NOLIMIT-NEXT: [[SHR32ri3:%[0-9]+]]:gr32 = SHR32ri [[COPY]], 2, implicit-def dead $eflags + ; NOLIMIT-NEXT: [[AND32ri2:%[0-9]+]]:gr32 = AND32ri [[SHR32ri3]], 7, implicit-def dead $eflags + ; NOLIMIT-NEXT: [[SUBREG_TO_REG2:%[0-9]+]]:gr64_nosp = SUBREG_TO_REG 0, [[AND32ri2]], %subreg.sub_32bit + ; NOLIMIT-NEXT: JMP64m $noreg, 8, [[SUBREG_TO_REG2]], %jump-table.1, $noreg + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: bb.4: + ; NOLIMIT-NEXT: successors: %bb.9(0x20000000), %bb.10(0x20000000), %bb.11(0x20000000), %bb.12(0x20000000) + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: [[MOV32rm2:%[0-9]+]]:gr32 = MOV32rm [[COPY1]], 1, $noreg, 0, $noreg + ; NOLIMIT-NEXT: [[SHR32ri4:%[0-9]+]]:gr32 = SHR32ri [[MOV32rm2]], 2, implicit-def dead $eflags + ; NOLIMIT-NEXT: [[SHR32ri5:%[0-9]+]]:gr32 = SHR32ri [[COPY]], 2, implicit-def dead $eflags + ; NOLIMIT-NEXT: [[AND32ri3:%[0-9]+]]:gr32 = AND32ri [[SHR32ri5]], 7, implicit-def dead $eflags + ; NOLIMIT-NEXT: [[SUBREG_TO_REG3:%[0-9]+]]:gr64_nosp = SUBREG_TO_REG 0, [[AND32ri3]], %subreg.sub_32bit + ; NOLIMIT-NEXT: JMP64m $noreg, 8, [[SUBREG_TO_REG3]], %jump-table.1, $noreg + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: bb.5: + ; NOLIMIT-NEXT: successors: %bb.9(0x20000000), %bb.10(0x20000000), %bb.11(0x20000000), %bb.12(0x20000000) + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: [[MOV32rm3:%[0-9]+]]:gr32 = MOV32rm [[COPY1]], 1, $noreg, 0, $noreg + ; NOLIMIT-NEXT: [[SHR32ri6:%[0-9]+]]:gr32 = SHR32ri [[MOV32rm3]], 3, implicit-def dead $eflags + ; NOLIMIT-NEXT: [[SHR32ri7:%[0-9]+]]:gr32 = SHR32ri [[COPY]], 2, implicit-def dead $eflags + ; NOLIMIT-NEXT: [[AND32ri4:%[0-9]+]]:gr32 = AND32ri [[SHR32ri7]], 7, implicit-def dead $eflags + ; NOLIMIT-NEXT: [[SUBREG_TO_REG4:%[0-9]+]]:gr64_nosp = SUBREG_TO_REG 0, [[AND32ri4]], %subreg.sub_32bit + ; NOLIMIT-NEXT: JMP64m $noreg, 8, [[SUBREG_TO_REG4]], %jump-table.1, $noreg + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: bb.6: + ; NOLIMIT-NEXT: successors: + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: bb.9: + ; NOLIMIT-NEXT: successors: %bb.13(0x80000000) + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: [[PHI:%[0-9]+]]:gr32 = PHI [[MOV32rm]], %bb.2, [[SHR32ri2]], %bb.3, [[SHR32ri4]], %bb.4, [[SHR32ri6]], %bb.5 + ; NOLIMIT-NEXT: [[MOV32rm4:%[0-9]+]]:gr32 = MOV32rm [[COPY1]], 1, $noreg, 0, $noreg + ; NOLIMIT-NEXT: JMP_1 %bb.13 + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: bb.10: + ; NOLIMIT-NEXT: successors: %bb.13(0x80000000) + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: [[PHI1:%[0-9]+]]:gr32 = PHI [[MOV32rm]], %bb.2, [[SHR32ri2]], %bb.3, [[SHR32ri4]], %bb.4, [[SHR32ri6]], %bb.5 + ; NOLIMIT-NEXT: [[MOV32rm5:%[0-9]+]]:gr32 = MOV32rm [[COPY1]], 1, $noreg, 0, $noreg + ; NOLIMIT-NEXT: [[SHR32ri8:%[0-9]+]]:gr32 = SHR32ri [[MOV32rm5]], 1, implicit-def dead $eflags + ; NOLIMIT-NEXT: JMP_1 %bb.13 + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: bb.11: + ; NOLIMIT-NEXT: successors: %bb.13(0x80000000) + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: [[PHI2:%[0-9]+]]:gr32 = PHI [[MOV32rm]], %bb.2, [[SHR32ri2]], %bb.3, [[SHR32ri4]], %bb.4, [[SHR32ri6]], %bb.5 + ; NOLIMIT-NEXT: [[MOV32rm6:%[0-9]+]]:gr32 = MOV32rm [[COPY1]], 1, $noreg, 0, $noreg + ; NOLIMIT-NEXT: [[SHR32ri9:%[0-9]+]]:gr32 = SHR32ri [[MOV32rm6]], 2, implicit-def dead $eflags + ; NOLIMIT-NEXT: JMP_1 %bb.13 + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: bb.12: + ; NOLIMIT-NEXT: successors: %bb.13(0x80000000) + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: [[PHI3:%[0-9]+]]:gr32 = PHI [[MOV32rm]], %bb.2, [[SHR32ri2]], %bb.3, [[SHR32ri4]], %bb.4, [[SHR32ri6]], %bb.5 + ; NOLIMIT-NEXT: [[MOV32rm7:%[0-9]+]]:gr32 = MOV32rm [[COPY1]], 1, $noreg, 0, $noreg + ; NOLIMIT-NEXT: [[SHR32ri10:%[0-9]+]]:gr32 = SHR32ri [[MOV32rm7]], 6, implicit-def dead $eflags + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: bb.13: + ; NOLIMIT-NEXT: [[PHI4:%[0-9]+]]:gr32 = PHI [[PHI]], %bb.9, [[PHI1]], %bb.10, [[PHI2]], %bb.11, [[PHI3]], %bb.12 + ; NOLIMIT-NEXT: [[PHI5:%[0-9]+]]:gr32 = PHI [[SHR32ri10]], %bb.12, [[SHR32ri9]], %bb.11, [[SHR32ri8]], %bb.10, [[MOV32rm4]], %bb.9 + ; NOLIMIT-NEXT: [[OR32rr:%[0-9]+]]:gr32 = OR32rr [[PHI5]], [[PHI4]], implicit-def dead $eflags + ; NOLIMIT-NEXT: $eax = COPY [[OR32rr]] + ; NOLIMIT-NEXT: RET 0, $eax + bb.0: + liveins: $rdi, $esi + + %11:gr32 = COPY $esi + %10:gr64 = COPY $rdi + %13:gr32 = SHR32ri %11, 1, implicit-def dead $eflags + %14:gr32 = AND32ri %13, 7, implicit-def dead $eflags + %12:gr64_nosp = SUBREG_TO_REG 0, killed %14, %subreg.sub_32bit + + bb.1: + successors: %bb.2, %bb.3, %bb.4, %bb.5 + + JMP64m $noreg, 8, %12, %jump-table.0, $noreg + + bb.2: + %0:gr32 = MOV32rm %10, 1, $noreg, 0, $noreg + JMP_1 %bb.7 + + bb.3: + %17:gr32 = MOV32rm %10, 1, $noreg, 0, $noreg + %1:gr32 = SHR32ri %17, 1, implicit-def dead $eflags + JMP_1 %bb.7 + + bb.4: + %16:gr32 = MOV32rm %10, 1, $noreg, 0, $noreg + %2:gr32 = SHR32ri %16, 2, implicit-def dead $eflags + JMP_1 %bb.7 + + bb.5: + %15:gr32 = MOV32rm %10, 1, $noreg, 0, $noreg + %3:gr32 = SHR32ri %15, 3, implicit-def dead $eflags + JMP_1 %bb.7 + + bb.6: + successors: + + bb.7: + %4:gr32 = PHI %3, %bb.5, %2, %bb.4, %1, %bb.3, %0, %bb.2 + %19:gr32 = SHR32ri %11, 2, implicit-def dead $eflags + %20:gr32 = AND32ri %19, 7, implicit-def dead $eflags + %18:gr64_nosp = SUBREG_TO_REG 0, killed %20, %subreg.sub_32bit + + bb.8: + successors: %bb.9, %bb.10, %bb.11, %bb.12 + + JMP64m $noreg, 8, %18, %jump-table.1, $noreg + + bb.9: + %5:gr32 = MOV32rm %10, 1, $noreg, 0, $noreg + JMP_1 %bb.13 + + bb.10: + %23:gr32 = MOV32rm %10, 1, $noreg, 0, $noreg + %6:gr32 = SHR32ri %23, 1, implicit-def dead $eflags + JMP_1 %bb.13 + + bb.11: + %22:gr32 = MOV32rm %10, 1, $noreg, 0, $noreg + %7:gr32 = SHR32ri %22, 2, implicit-def dead $eflags + JMP_1 %bb.13 + + bb.12: + %21:gr32 = MOV32rm %10, 1, $noreg, 0, $noreg + %8:gr32 = SHR32ri %21, 6, implicit-def dead $eflags + + bb.13: + %9:gr32 = PHI %8, %bb.12, %7, %bb.11, %6, %bb.10, %5, %bb.9 + %24:gr32 = OR32rr %9, %4, implicit-def dead $eflags + $eax = COPY %24 + RET 0, $eax + +... -- GitLab From 915c84b1480bb3c6d2e44ca83822d2c2304b763a Mon Sep 17 00:00:00 2001 From: Pavel Labath Date: Wed, 17 Apr 2024 14:33:18 +0200 Subject: [PATCH 044/862] [lldb] Fix evaluation of expressions with static initializers (#89063) After 281d71604f418eb952e967d9dc4b26241b7f96a, llvm generates 32-bit relocations, which overflow when we load these objects into high memory. Interestingly, setting the code model to "large" does not help here (perhaps it is the default?). I'm not completely sure that this is the right thing to do, but it doesn't seem to cause any ill effects. I'll follow up with the author of that patch about the expected behavior here. --- lldb/source/Expression/IRExecutionUnit.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lldb/source/Expression/IRExecutionUnit.cpp b/lldb/source/Expression/IRExecutionUnit.cpp index cb9bee8733e1..7ad0e5ff22b2 100644 --- a/lldb/source/Expression/IRExecutionUnit.cpp +++ b/lldb/source/Expression/IRExecutionUnit.cpp @@ -13,6 +13,7 @@ #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" +#include "llvm/Support/CodeGen.h" #include "llvm/Support/SourceMgr.h" #include "llvm/Support/raw_ostream.h" @@ -279,10 +280,13 @@ void IRExecutionUnit::GetRunnableInfo(Status &error, lldb::addr_t &func_addr, llvm::EngineBuilder builder(std::move(m_module_up)); llvm::Triple triple(m_module->getTargetTriple()); + // PIC needed for ELF to avoid generating 32-bit relocations (which overflow + // if the object is loaded into high memory). + bool want_pic = triple.isOSBinFormatMachO() || triple.isOSBinFormatELF(); + builder.setEngineKind(llvm::EngineKind::JIT) .setErrorStr(&error_string) - .setRelocationModel(triple.isOSBinFormatMachO() ? llvm::Reloc::PIC_ - : llvm::Reloc::Static) + .setRelocationModel(want_pic ? llvm::Reloc::PIC_ : llvm::Reloc::Static) .setMCJITMemoryManager(std::make_unique(*this)) .setOptLevel(llvm::CodeGenOptLevel::Less); -- GitLab From 79726ef5d20f3bf6510b67d7dd3378b8a41db768 Mon Sep 17 00:00:00 2001 From: "Kevin P. Neal" Date: Wed, 17 Apr 2024 08:34:25 -0400 Subject: [PATCH 045/862] [VP] Correct lowering of predicated fma and faddmul to avoid strictfp. (#85272) Correct missing cases in a switch that result in @llvm.vp.fma.v4f32 getting lowered to a constrained fma intrinsic. Vector predicated lowering to contrained intrinsics is not supported currently, and there's no consensus on the path forward. We certainly shouldn't be introducing constrained intrinsics into a function that isn't strictfp. Problem found with D146845. --- llvm/include/llvm/IR/Intrinsics.h | 4 + llvm/lib/CodeGen/ExpandVectorPredication.cpp | 12 +- llvm/lib/IR/Function.cpp | 22 ++- .../Generic/expand-vp-fp-intrinsics.ll | 176 ++++++++++++++++++ 4 files changed, 203 insertions(+), 11 deletions(-) create mode 100644 llvm/test/CodeGen/Generic/expand-vp-fp-intrinsics.ll diff --git a/llvm/include/llvm/IR/Intrinsics.h b/llvm/include/llvm/IR/Intrinsics.h index 0dfe9f029f9b..92eae344ce72 100644 --- a/llvm/include/llvm/IR/Intrinsics.h +++ b/llvm/include/llvm/IR/Intrinsics.h @@ -105,6 +105,10 @@ namespace Intrinsic { /// Map a MS builtin name to an intrinsic ID. ID getIntrinsicForMSBuiltin(const char *Prefix, StringRef BuiltinName); + /// Returns true if the intrinsic ID is for one of the "Constrained + /// Floating-Point Intrinsics". + bool isConstrainedFPIntrinsic(ID QID); + /// This is a type descriptor which explains the type requirements of an /// intrinsic. This is returned by getIntrinsicInfoTableEntries. struct IITDescriptor { diff --git a/llvm/lib/CodeGen/ExpandVectorPredication.cpp b/llvm/lib/CodeGen/ExpandVectorPredication.cpp index 0fe4cfefdb16..8e623c85b737 100644 --- a/llvm/lib/CodeGen/ExpandVectorPredication.cpp +++ b/llvm/lib/CodeGen/ExpandVectorPredication.cpp @@ -340,6 +340,8 @@ Value *CachingVPExpander::expandPredicationToFPCall( replaceOperation(*NewOp, VPI); return NewOp; } + case Intrinsic::fma: + case Intrinsic::fmuladd: case Intrinsic::experimental_constrained_fma: case Intrinsic::experimental_constrained_fmuladd: { Value *Op0 = VPI.getOperand(0); @@ -347,8 +349,12 @@ Value *CachingVPExpander::expandPredicationToFPCall( Value *Op2 = VPI.getOperand(2); Function *Fn = Intrinsic::getDeclaration( VPI.getModule(), UnpredicatedIntrinsicID, {VPI.getType()}); - Value *NewOp = - Builder.CreateConstrainedFPCall(Fn, {Op0, Op1, Op2}, VPI.getName()); + Value *NewOp; + if (Intrinsic::isConstrainedFPIntrinsic(UnpredicatedIntrinsicID)) + NewOp = + Builder.CreateConstrainedFPCall(Fn, {Op0, Op1, Op2}, VPI.getName()); + else + NewOp = Builder.CreateCall(Fn, {Op0, Op1, Op2}, VPI.getName()); replaceOperation(*NewOp, VPI); return NewOp; } @@ -731,6 +737,8 @@ Value *CachingVPExpander::expandPredication(VPIntrinsic &VPI) { case Intrinsic::vp_minnum: case Intrinsic::vp_maximum: case Intrinsic::vp_minimum: + case Intrinsic::vp_fma: + case Intrinsic::vp_fmuladd: return expandPredicationToFPCall(Builder, VPI, VPI.getFunctionalIntrinsicID().value()); case Intrinsic::vp_load: diff --git a/llvm/lib/IR/Function.cpp b/llvm/lib/IR/Function.cpp index 96953ac49c19..818a167560de 100644 --- a/llvm/lib/IR/Function.cpp +++ b/llvm/lib/IR/Function.cpp @@ -499,15 +499,7 @@ static MutableArrayRef makeArgArray(Argument *Args, size_t Count) { } bool Function::isConstrainedFPIntrinsic() const { - switch (getIntrinsicID()) { -#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \ - case Intrinsic::INTRINSIC: -#include "llvm/IR/ConstrainedOps.def" - return true; -#undef INSTRUCTION - default: - return false; - } + return Intrinsic::isConstrainedFPIntrinsic(getIntrinsicID()); } void Function::clearArguments() { @@ -1486,6 +1478,18 @@ Function *Intrinsic::getDeclaration(Module *M, ID id, ArrayRef Tys) { #include "llvm/IR/IntrinsicImpl.inc" #undef GET_LLVM_INTRINSIC_FOR_MS_BUILTIN +bool Intrinsic::isConstrainedFPIntrinsic(ID QID) { + switch (QID) { +#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \ + case Intrinsic::INTRINSIC: +#include "llvm/IR/ConstrainedOps.def" + return true; +#undef INSTRUCTION + default: + return false; + } +} + using DeferredIntrinsicMatchPair = std::pair>; diff --git a/llvm/test/CodeGen/Generic/expand-vp-fp-intrinsics.ll b/llvm/test/CodeGen/Generic/expand-vp-fp-intrinsics.ll new file mode 100644 index 000000000000..bc89ddea6b85 --- /dev/null +++ b/llvm/test/CodeGen/Generic/expand-vp-fp-intrinsics.ll @@ -0,0 +1,176 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -expandvp -S < %s | FileCheck %s + +define void @vp_fadd_v4f32(<4 x float> %a0, <4 x float> %a1, ptr %out, i32 %vp) nounwind { +; CHECK-LABEL: define void @vp_fadd_v4f32( +; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], ptr [[OUT:%.*]], i32 [[VP:%.*]]) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: [[RES1:%.*]] = fadd <4 x float> [[A0]], [[A1]] +; CHECK-NEXT: store <4 x float> [[RES1]], ptr [[OUT]], align 16 +; CHECK-NEXT: ret void +; + %res = call <4 x float> @llvm.vp.fadd.v4f32(<4 x float> %a0, <4 x float> %a1, <4 x i1> , i32 %vp) + store <4 x float> %res, ptr %out + ret void +} +declare <4 x float> @llvm.vp.fadd.v4f32(<4 x float>, <4 x float>, <4 x i1>, i32) + +define void @vp_fsub_v4f32(<4 x float> %a0, <4 x float> %a1, ptr %out, i32 %vp) nounwind { +; CHECK-LABEL: define void @vp_fsub_v4f32( +; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], ptr [[OUT:%.*]], i32 [[VP:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[RES1:%.*]] = fsub <4 x float> [[A0]], [[A1]] +; CHECK-NEXT: store <4 x float> [[RES1]], ptr [[OUT]], align 16 +; CHECK-NEXT: ret void +; + %res = call <4 x float> @llvm.vp.fsub.v4f32(<4 x float> %a0, <4 x float> %a1, <4 x i1> , i32 %vp) + store <4 x float> %res, ptr %out + ret void +} +declare <4 x float> @llvm.vp.fsub.v4f32(<4 x float>, <4 x float>, <4 x i1>, i32) + +define void @vp_fmul_v4f32(<4 x float> %a0, <4 x float> %a1, ptr %out, i32 %vp) nounwind { +; CHECK-LABEL: define void @vp_fmul_v4f32( +; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], ptr [[OUT:%.*]], i32 [[VP:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[RES1:%.*]] = fmul <4 x float> [[A0]], [[A1]] +; CHECK-NEXT: store <4 x float> [[RES1]], ptr [[OUT]], align 16 +; CHECK-NEXT: ret void +; + %res = call <4 x float> @llvm.vp.fmul.v4f32(<4 x float> %a0, <4 x float> %a1, <4 x i1> , i32 %vp) + store <4 x float> %res, ptr %out + ret void +} +declare <4 x float> @llvm.vp.fmul.v4f32(<4 x float>, <4 x float>, <4 x i1>, i32) + +define void @vp_fdiv_v4f32(<4 x float> %a0, <4 x float> %a1, ptr %out, i32 %vp) nounwind { +; CHECK-LABEL: define void @vp_fdiv_v4f32( +; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], ptr [[OUT:%.*]], i32 [[VP:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[RES1:%.*]] = fdiv <4 x float> [[A0]], [[A1]] +; CHECK-NEXT: store <4 x float> [[RES1]], ptr [[OUT]], align 16 +; CHECK-NEXT: ret void +; + %res = call <4 x float> @llvm.vp.fdiv.v4f32(<4 x float> %a0, <4 x float> %a1, <4 x i1> , i32 %vp) + store <4 x float> %res, ptr %out + ret void +} +declare <4 x float> @llvm.vp.fdiv.v4f32(<4 x float>, <4 x float>, <4 x i1>, i32) + +define void @vp_frem_v4f32(<4 x float> %a0, <4 x float> %a1, ptr %out, i32 %vp) nounwind { +; CHECK-LABEL: define void @vp_frem_v4f32( +; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], ptr [[OUT:%.*]], i32 [[VP:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[RES1:%.*]] = frem <4 x float> [[A0]], [[A1]] +; CHECK-NEXT: store <4 x float> [[RES1]], ptr [[OUT]], align 16 +; CHECK-NEXT: ret void +; + %res = call <4 x float> @llvm.vp.frem.v4f32(<4 x float> %a0, <4 x float> %a1, <4 x i1> , i32 %vp) + store <4 x float> %res, ptr %out + ret void +} +declare <4 x float> @llvm.vp.frem.v4f32(<4 x float>, <4 x float>, <4 x i1>, i32) + +define void @vp_fabs_v4f32(<4 x float> %a0, <4 x float> %a1, ptr %out, i32 %vp) nounwind { +; CHECK-LABEL: define void @vp_fabs_v4f32( +; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], ptr [[OUT:%.*]], i32 [[VP:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[RES1:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[A0]]) +; CHECK-NEXT: store <4 x float> [[RES1]], ptr [[OUT]], align 16 +; CHECK-NEXT: ret void +; + %res = call <4 x float> @llvm.vp.fabs.v4f32(<4 x float> %a0, <4 x i1> , i32 %vp) + store <4 x float> %res, ptr %out + ret void +} +declare <4 x float> @llvm.vp.fabs.v4f32(<4 x float>, <4 x i1>, i32) + +define void @vp_sqrt_v4f32(<4 x float> %a0, <4 x float> %a1, ptr %out, i32 %vp) nounwind { +; CHECK-LABEL: define void @vp_sqrt_v4f32( +; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], ptr [[OUT:%.*]], i32 [[VP:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[RES1:%.*]] = call <4 x float> @llvm.sqrt.v4f32(<4 x float> [[A0]]) +; CHECK-NEXT: store <4 x float> [[RES1]], ptr [[OUT]], align 16 +; CHECK-NEXT: ret void +; + %res = call <4 x float> @llvm.vp.sqrt.v4f32(<4 x float> %a0, <4 x i1> , i32 %vp) + store <4 x float> %res, ptr %out + ret void +} +declare <4 x float> @llvm.vp.sqrt.v4f32(<4 x float>, <4 x i1>, i32) + +define void @vp_fneg_v4f32(<4 x float> %a0, <4 x float> %a1, ptr %out, i32 %vp) nounwind { +; CHECK-LABEL: define void @vp_fneg_v4f32( +; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], ptr [[OUT:%.*]], i32 [[VP:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[RES1:%.*]] = fneg <4 x float> [[A0]] +; CHECK-NEXT: store <4 x float> [[RES1]], ptr [[OUT]], align 16 +; CHECK-NEXT: ret void +; + %res = call <4 x float> @llvm.vp.fneg.v4f32(<4 x float> %a0, <4 x i1> , i32 %vp) + store <4 x float> %res, ptr %out + ret void +} +declare <4 x float> @llvm.vp.fneg.v4f32(<4 x float>, <4 x i1>, i32) + +define void @vp_fma_v4f32(<4 x float> %a0, <4 x float> %a1, ptr %out, i4 %a5) nounwind { +; CHECK-LABEL: define void @vp_fma_v4f32( +; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], ptr [[OUT:%.*]], i4 [[A5:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[RES1:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[A0]], <4 x float> [[A1]], <4 x float> [[A1]]) +; CHECK-NEXT: store <4 x float> [[RES1]], ptr [[OUT]], align 16 +; CHECK-NEXT: ret void +; + %res = call <4 x float> @llvm.vp.fma.v4f32(<4 x float> %a0, <4 x float> %a1, <4 x float> %a1, <4 x i1> , i32 4) + store <4 x float> %res, ptr %out + ret void +} +declare <4 x float> @llvm.vp.fma.v4f32(<4 x float>, <4 x float>, <4 x float>, <4 x i1>, i32) + +define void @vp_fmuladd_v4f32(<4 x float> %a0, <4 x float> %a1, ptr %out, i4 %a5) nounwind { +; CHECK-LABEL: define void @vp_fmuladd_v4f32( +; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], ptr [[OUT:%.*]], i4 [[A5:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[RES1:%.*]] = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> [[A0]], <4 x float> [[A1]], <4 x float> [[A1]]) +; CHECK-NEXT: store <4 x float> [[RES1]], ptr [[OUT]], align 16 +; CHECK-NEXT: ret void +; + %res = call <4 x float> @llvm.vp.fmuladd.v4f32(<4 x float> %a0, <4 x float> %a1, <4 x float> %a1, <4 x i1> , i32 4) + store <4 x float> %res, ptr %out + ret void +} +declare <4 x float> @llvm.vp.fmuladd.v4f32(<4 x float>, <4 x float>, <4 x float>, <4 x i1>, i32) + +declare <4 x float> @llvm.vp.maxnum.v4f32(<4 x float>, <4 x float>, <4 x i1>, i32) +define <4 x float> @vfmax_vv_v4f32(<4 x float> %va, <4 x float> %vb, <4 x i1> %m, i32 zeroext %evl) { +; CHECK-LABEL: define <4 x float> @vfmax_vv_v4f32( +; CHECK-SAME: <4 x float> [[VA:%.*]], <4 x float> [[VB:%.*]], <4 x i1> [[M:%.*]], i32 zeroext [[EVL:%.*]]) { +; CHECK-NEXT: [[V1:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VA]], <4 x float> [[VB]]) +; CHECK-NEXT: ret <4 x float> [[V1]] +; + %v = call <4 x float> @llvm.vp.maxnum.v4f32(<4 x float> %va, <4 x float> %vb, <4 x i1> %m, i32 %evl) + ret <4 x float> %v +} + +declare <8 x float> @llvm.vp.maxnum.v8f32(<8 x float>, <8 x float>, <8 x i1>, i32) +define <8 x float> @vfmax_vv_v8f32(<8 x float> %va, <8 x float> %vb, <8 x i1> %m, i32 zeroext %evl) { +; CHECK-LABEL: define <8 x float> @vfmax_vv_v8f32( +; CHECK-SAME: <8 x float> [[VA:%.*]], <8 x float> [[VB:%.*]], <8 x i1> [[M:%.*]], i32 zeroext [[EVL:%.*]]) { +; CHECK-NEXT: [[V1:%.*]] = call <8 x float> @llvm.maxnum.v8f32(<8 x float> [[VA]], <8 x float> [[VB]]) +; CHECK-NEXT: ret <8 x float> [[V1]] +; + %v = call <8 x float> @llvm.vp.maxnum.v8f32(<8 x float> %va, <8 x float> %vb, <8 x i1> %m, i32 %evl) + ret <8 x float> %v +} + +declare <4 x float> @llvm.vp.minnum.v4f32(<4 x float>, <4 x float>, <4 x i1>, i32) +define <4 x float> @vfmin_vv_v4f32(<4 x float> %va, <4 x float> %vb, <4 x i1> %m, i32 zeroext %evl) { +; CHECK-LABEL: define <4 x float> @vfmin_vv_v4f32( +; CHECK-SAME: <4 x float> [[VA:%.*]], <4 x float> [[VB:%.*]], <4 x i1> [[M:%.*]], i32 zeroext [[EVL:%.*]]) { +; CHECK-NEXT: [[V1:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VA]], <4 x float> [[VB]]) +; CHECK-NEXT: ret <4 x float> [[V1]] +; + %v = call <4 x float> @llvm.vp.minnum.v4f32(<4 x float> %va, <4 x float> %vb, <4 x i1> %m, i32 %evl) + ret <4 x float> %v +} + +declare <8 x float> @llvm.vp.minnum.v8f32(<8 x float>, <8 x float>, <8 x i1>, i32) +define <8 x float> @vfmin_vv_v8f32(<8 x float> %va, <8 x float> %vb, <8 x i1> %m, i32 zeroext %evl) { +; CHECK-LABEL: define <8 x float> @vfmin_vv_v8f32( +; CHECK-SAME: <8 x float> [[VA:%.*]], <8 x float> [[VB:%.*]], <8 x i1> [[M:%.*]], i32 zeroext [[EVL:%.*]]) { +; CHECK-NEXT: [[V1:%.*]] = call <8 x float> @llvm.minnum.v8f32(<8 x float> [[VA]], <8 x float> [[VB]]) +; CHECK-NEXT: ret <8 x float> [[V1]] +; + %v = call <8 x float> @llvm.vp.minnum.v8f32(<8 x float> %va, <8 x float> %vb, <8 x i1> %m, i32 %evl) + ret <8 x float> %v +} -- GitLab From 7b8625ec16efcb6d11e14a80e08c65dbfe68db9f Mon Sep 17 00:00:00 2001 From: Fabian Ritter Date: Wed, 17 Apr 2024 14:54:14 +0200 Subject: [PATCH 046/862] [AMDGPU][Docs] Fix broken link to HRF memory model reference (#88696) The link to the Heterogeneous-race-free Memory Models ASPLOS'14 paper by Hower et al. pointed to a bogus website, probably because the domain ownership has changed. This patch updates it to a version hosted on research.cs.wisc.edu. --- llvm/docs/AMDGPUUsage.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst index 22c1d1f186ea..7da5d8e41f6f 100644 --- a/llvm/docs/AMDGPUUsage.rst +++ b/llvm/docs/AMDGPUUsage.rst @@ -16024,7 +16024,7 @@ Additional Documentation .. [CLANG-ATTR] `Attributes in Clang `__ .. [DWARF] `DWARF Debugging Information Format `__ .. [ELF] `Executable and Linkable Format (ELF) `__ -.. [HRF] `Heterogeneous-race-free Memory Models `__ +.. [HRF] `Heterogeneous-race-free Memory Models `__ .. [HSA] `Heterogeneous System Architecture (HSA) Foundation `__ .. [MsgPack] `Message Pack `__ .. [OpenCL] `The OpenCL Specification Version 2.0 `__ -- GitLab From 971ec1f0eea324d4a1eec6709e2c97e1798a6002 Mon Sep 17 00:00:00 2001 From: DianQK Date: Wed, 17 Apr 2024 20:54:17 +0800 Subject: [PATCH 047/862] [Inline] Regenerate inline-switch-default-2.ll (NFC) --- .../Inline/inline-switch-default-2.ll | 176 ------------------ 1 file changed, 176 deletions(-) diff --git a/llvm/test/Transforms/Inline/inline-switch-default-2.ll b/llvm/test/Transforms/Inline/inline-switch-default-2.ll index 8d3e24c798df..82dae1c27648 100644 --- a/llvm/test/Transforms/Inline/inline-switch-default-2.ll +++ b/llvm/test/Transforms/Inline/inline-switch-default-2.ll @@ -4,50 +4,6 @@ ; Check for scenarios without TTI. define i64 @foo1(i64 %a) { -; LOOKUPTABLE-LABEL: define i64 @foo1( -; LOOKUPTABLE-SAME: i64 [[TMP0:%.*]]) { -; LOOKUPTABLE-NEXT: switch i64 [[TMP0]], label [[DEFAULT_BRANCH_I:%.*]] [ -; LOOKUPTABLE-NEXT: i64 0, label [[BRANCH_0_I:%.*]] -; LOOKUPTABLE-NEXT: i64 2, label [[BRANCH_2_I:%.*]] -; LOOKUPTABLE-NEXT: i64 4, label [[BRANCH_4_I:%.*]] -; LOOKUPTABLE-NEXT: i64 6, label [[BRANCH_6_I:%.*]] -; LOOKUPTABLE-NEXT: ] -; LOOKUPTABLE: branch_0.i: -; LOOKUPTABLE-NEXT: br label [[BAR1_EXIT:%.*]] -; LOOKUPTABLE: branch_2.i: -; LOOKUPTABLE-NEXT: br label [[BAR1_EXIT]] -; LOOKUPTABLE: branch_4.i: -; LOOKUPTABLE-NEXT: br label [[BAR1_EXIT]] -; LOOKUPTABLE: branch_6.i: -; LOOKUPTABLE-NEXT: br label [[BAR1_EXIT]] -; LOOKUPTABLE: default_branch.i: -; LOOKUPTABLE-NEXT: br label [[BAR1_EXIT]] -; LOOKUPTABLE: bar1.exit: -; LOOKUPTABLE-NEXT: [[TMP2:%.*]] = phi i64 [ 5, [[BRANCH_0_I]] ], [ 9, [[BRANCH_2_I]] ], [ 2, [[BRANCH_4_I]] ], [ 7, [[BRANCH_6_I]] ], [ 3, [[DEFAULT_BRANCH_I]] ] -; LOOKUPTABLE-NEXT: ret i64 [[TMP2]] -; -; SWITCH-LABEL: define i64 @foo1( -; SWITCH-SAME: i64 [[TMP0:%.*]]) { -; SWITCH-NEXT: switch i64 [[TMP0]], label [[DEFAULT_BRANCH_I:%.*]] [ -; SWITCH-NEXT: i64 0, label [[BRANCH_0_I:%.*]] -; SWITCH-NEXT: i64 2, label [[BRANCH_2_I:%.*]] -; SWITCH-NEXT: i64 4, label [[BRANCH_4_I:%.*]] -; SWITCH-NEXT: i64 6, label [[BRANCH_6_I:%.*]] -; SWITCH-NEXT: ] -; SWITCH: branch_0.i: -; SWITCH-NEXT: br label [[BAR1_EXIT:%.*]] -; SWITCH: branch_2.i: -; SWITCH-NEXT: br label [[BAR1_EXIT]] -; SWITCH: branch_4.i: -; SWITCH-NEXT: br label [[BAR1_EXIT]] -; SWITCH: branch_6.i: -; SWITCH-NEXT: br label [[BAR1_EXIT]] -; SWITCH: default_branch.i: -; SWITCH-NEXT: br label [[BAR1_EXIT]] -; SWITCH: bar1.exit: -; SWITCH-NEXT: [[TMP2:%.*]] = phi i64 [ 5, [[BRANCH_0_I]] ], [ 9, [[BRANCH_2_I]] ], [ 2, [[BRANCH_4_I]] ], [ 7, [[BRANCH_6_I]] ], [ 3, [[DEFAULT_BRANCH_I]] ] -; SWITCH-NEXT: ret i64 [[TMP2]] -; ; CHECK-LABEL: define i64 @foo1( ; CHECK-SAME: i64 [[A:%.*]]) { ; CHECK-NEXT: [[B:%.*]] = call i64 @bar1(i64 [[A]]) @@ -58,50 +14,6 @@ define i64 @foo1(i64 %a) { } define i64 @foo2(i64 %a) { -; LOOKUPTABLE-LABEL: define i64 @foo2( -; LOOKUPTABLE-SAME: i64 [[TMP0:%.*]]) { -; LOOKUPTABLE-NEXT: switch i64 [[TMP0]], label [[UNREACHABLEDEFAULT_I:%.*]] [ -; LOOKUPTABLE-NEXT: i64 0, label [[BRANCH_0_I:%.*]] -; LOOKUPTABLE-NEXT: i64 2, label [[BRANCH_2_I:%.*]] -; LOOKUPTABLE-NEXT: i64 4, label [[BRANCH_4_I:%.*]] -; LOOKUPTABLE-NEXT: i64 6, label [[BRANCH_6_I:%.*]] -; LOOKUPTABLE-NEXT: ] -; LOOKUPTABLE: branch_0.i: -; LOOKUPTABLE-NEXT: br label [[BAR2_EXIT:%.*]] -; LOOKUPTABLE: branch_2.i: -; LOOKUPTABLE-NEXT: br label [[BAR2_EXIT]] -; LOOKUPTABLE: branch_4.i: -; LOOKUPTABLE-NEXT: br label [[BAR2_EXIT]] -; LOOKUPTABLE: branch_6.i: -; LOOKUPTABLE-NEXT: br label [[BAR2_EXIT]] -; LOOKUPTABLE: unreachabledefault.i: -; LOOKUPTABLE-NEXT: unreachable -; LOOKUPTABLE: bar2.exit: -; LOOKUPTABLE-NEXT: [[TMP2:%.*]] = phi i64 [ 5, [[BRANCH_0_I]] ], [ 9, [[BRANCH_2_I]] ], [ 2, [[BRANCH_4_I]] ], [ 7, [[BRANCH_6_I]] ] -; LOOKUPTABLE-NEXT: ret i64 [[TMP2]] -; -; SWITCH-LABEL: define i64 @foo2( -; SWITCH-SAME: i64 [[TMP0:%.*]]) { -; SWITCH-NEXT: switch i64 [[TMP0]], label [[UNREACHABLEDEFAULT_I:%.*]] [ -; SWITCH-NEXT: i64 0, label [[BRANCH_0_I:%.*]] -; SWITCH-NEXT: i64 2, label [[BRANCH_2_I:%.*]] -; SWITCH-NEXT: i64 4, label [[BRANCH_4_I:%.*]] -; SWITCH-NEXT: i64 6, label [[BRANCH_6_I:%.*]] -; SWITCH-NEXT: ] -; SWITCH: branch_0.i: -; SWITCH-NEXT: br label [[BAR2_EXIT:%.*]] -; SWITCH: branch_2.i: -; SWITCH-NEXT: br label [[BAR2_EXIT]] -; SWITCH: branch_4.i: -; SWITCH-NEXT: br label [[BAR2_EXIT]] -; SWITCH: branch_6.i: -; SWITCH-NEXT: br label [[BAR2_EXIT]] -; SWITCH: unreachabledefault.i: -; SWITCH-NEXT: unreachable -; SWITCH: bar2.exit: -; SWITCH-NEXT: [[TMP2:%.*]] = phi i64 [ 5, [[BRANCH_0_I]] ], [ 9, [[BRANCH_2_I]] ], [ 2, [[BRANCH_4_I]] ], [ 7, [[BRANCH_6_I]] ] -; SWITCH-NEXT: ret i64 [[TMP2]] -; ; CHECK-LABEL: define i64 @foo2( ; CHECK-SAME: i64 [[A:%.*]]) { ; CHECK-NEXT: switch i64 [[A]], label [[UNREACHABLEDEFAULT_I:%.*]] [ @@ -129,50 +41,6 @@ define i64 @foo2(i64 %a) { } define i64 @bar1(i64 %a) { -; LOOKUPTABLE-LABEL: define i64 @bar1( -; LOOKUPTABLE-SAME: i64 [[TMP0:%.*]]) { -; LOOKUPTABLE-NEXT: switch i64 [[TMP0]], label [[DEFAULT_BRANCH:%.*]] [ -; LOOKUPTABLE-NEXT: i64 0, label [[BRANCH_0:%.*]] -; LOOKUPTABLE-NEXT: i64 2, label [[BRANCH_2:%.*]] -; LOOKUPTABLE-NEXT: i64 4, label [[BRANCH_4:%.*]] -; LOOKUPTABLE-NEXT: i64 6, label [[BRANCH_6:%.*]] -; LOOKUPTABLE-NEXT: ] -; LOOKUPTABLE: branch_0: -; LOOKUPTABLE-NEXT: br label [[EXIT:%.*]] -; LOOKUPTABLE: branch_2: -; LOOKUPTABLE-NEXT: br label [[EXIT]] -; LOOKUPTABLE: branch_4: -; LOOKUPTABLE-NEXT: br label [[EXIT]] -; LOOKUPTABLE: branch_6: -; LOOKUPTABLE-NEXT: br label [[EXIT]] -; LOOKUPTABLE: default_branch: -; LOOKUPTABLE-NEXT: br label [[EXIT]] -; LOOKUPTABLE: exit: -; LOOKUPTABLE-NEXT: [[TMP2:%.*]] = phi i64 [ 5, [[BRANCH_0]] ], [ 9, [[BRANCH_2]] ], [ 2, [[BRANCH_4]] ], [ 7, [[BRANCH_6]] ], [ 3, [[DEFAULT_BRANCH]] ] -; LOOKUPTABLE-NEXT: ret i64 [[TMP2]] -; -; SWITCH-LABEL: define i64 @bar1( -; SWITCH-SAME: i64 [[TMP0:%.*]]) { -; SWITCH-NEXT: switch i64 [[TMP0]], label [[DEFAULT_BRANCH:%.*]] [ -; SWITCH-NEXT: i64 0, label [[BRANCH_0:%.*]] -; SWITCH-NEXT: i64 2, label [[BRANCH_2:%.*]] -; SWITCH-NEXT: i64 4, label [[BRANCH_4:%.*]] -; SWITCH-NEXT: i64 6, label [[BRANCH_6:%.*]] -; SWITCH-NEXT: ] -; SWITCH: branch_0: -; SWITCH-NEXT: br label [[EXIT:%.*]] -; SWITCH: branch_2: -; SWITCH-NEXT: br label [[EXIT]] -; SWITCH: branch_4: -; SWITCH-NEXT: br label [[EXIT]] -; SWITCH: branch_6: -; SWITCH-NEXT: br label [[EXIT]] -; SWITCH: default_branch: -; SWITCH-NEXT: br label [[EXIT]] -; SWITCH: exit: -; SWITCH-NEXT: [[TMP2:%.*]] = phi i64 [ 5, [[BRANCH_0]] ], [ 9, [[BRANCH_2]] ], [ 2, [[BRANCH_4]] ], [ 7, [[BRANCH_6]] ], [ 3, [[DEFAULT_BRANCH]] ] -; SWITCH-NEXT: ret i64 [[TMP2]] -; ; CHECK-LABEL: define i64 @bar1( ; CHECK-SAME: i64 [[A:%.*]]) { ; CHECK-NEXT: switch i64 [[A]], label [[DEFAULT_BRANCH:%.*]] [ @@ -223,50 +91,6 @@ exit: } define i64 @bar2(i64 %a) { -; LOOKUPTABLE-LABEL: define i64 @bar2( -; LOOKUPTABLE-SAME: i64 [[TMP0:%.*]]) { -; LOOKUPTABLE-NEXT: switch i64 [[TMP0]], label [[UNREACHABLEDEFAULT:%.*]] [ -; LOOKUPTABLE-NEXT: i64 0, label [[BRANCH_0:%.*]] -; LOOKUPTABLE-NEXT: i64 2, label [[BRANCH_2:%.*]] -; LOOKUPTABLE-NEXT: i64 4, label [[BRANCH_4:%.*]] -; LOOKUPTABLE-NEXT: i64 6, label [[BRANCH_6:%.*]] -; LOOKUPTABLE-NEXT: ] -; LOOKUPTABLE: branch_0: -; LOOKUPTABLE-NEXT: br label [[EXIT:%.*]] -; LOOKUPTABLE: branch_2: -; LOOKUPTABLE-NEXT: br label [[EXIT]] -; LOOKUPTABLE: branch_4: -; LOOKUPTABLE-NEXT: br label [[EXIT]] -; LOOKUPTABLE: branch_6: -; LOOKUPTABLE-NEXT: br label [[EXIT]] -; LOOKUPTABLE: unreachabledefault: -; LOOKUPTABLE-NEXT: unreachable -; LOOKUPTABLE: exit: -; LOOKUPTABLE-NEXT: [[TMP2:%.*]] = phi i64 [ 5, [[BRANCH_0]] ], [ 9, [[BRANCH_2]] ], [ 2, [[BRANCH_4]] ], [ 7, [[BRANCH_6]] ] -; LOOKUPTABLE-NEXT: ret i64 [[TMP2]] -; -; SWITCH-LABEL: define i64 @bar2( -; SWITCH-SAME: i64 [[TMP0:%.*]]) { -; SWITCH-NEXT: switch i64 [[TMP0]], label [[UNREACHABLEDEFAULT:%.*]] [ -; SWITCH-NEXT: i64 0, label [[BRANCH_0:%.*]] -; SWITCH-NEXT: i64 2, label [[BRANCH_2:%.*]] -; SWITCH-NEXT: i64 4, label [[BRANCH_4:%.*]] -; SWITCH-NEXT: i64 6, label [[BRANCH_6:%.*]] -; SWITCH-NEXT: ] -; SWITCH: branch_0: -; SWITCH-NEXT: br label [[EXIT:%.*]] -; SWITCH: branch_2: -; SWITCH-NEXT: br label [[EXIT]] -; SWITCH: branch_4: -; SWITCH-NEXT: br label [[EXIT]] -; SWITCH: branch_6: -; SWITCH-NEXT: br label [[EXIT]] -; SWITCH: unreachabledefault: -; SWITCH-NEXT: unreachable -; SWITCH: exit: -; SWITCH-NEXT: [[TMP2:%.*]] = phi i64 [ 5, [[BRANCH_0]] ], [ 9, [[BRANCH_2]] ], [ 2, [[BRANCH_4]] ], [ 7, [[BRANCH_6]] ] -; SWITCH-NEXT: ret i64 [[TMP2]] -; ; CHECK-LABEL: define i64 @bar2( ; CHECK-SAME: i64 [[A:%.*]]) { ; CHECK-NEXT: switch i64 [[A]], label [[UNREACHABLEDEFAULT:%.*]] [ -- GitLab From 73140daebbf522dbb14dc4b2f3c67dc0aa1a62dd Mon Sep 17 00:00:00 2001 From: "Oleksandr \"Alex\" Zinenko" Date: Wed, 17 Apr 2024 15:01:59 +0200 Subject: [PATCH 048/862] [mlir] expose transform dialect symbol merge to python (#87690) This functionality is available in C++, make it available in Python directly to operate on transform modules. --- .../mlir-c/Dialect/Transform/Interpreter.h | 12 ++- .../Bindings/Python/TransformInterpreter.cpp | 15 ++++ .../lib/CAPI/Dialect/TransformInterpreter.cpp | 9 +++ .../transform/interpreter/__init__.py | 10 ++- .../python/dialects/transform_interpreter.py | 76 +++++++++++++++++++ 5 files changed, 120 insertions(+), 2 deletions(-) diff --git a/mlir/include/mlir-c/Dialect/Transform/Interpreter.h b/mlir/include/mlir-c/Dialect/Transform/Interpreter.h index 00095d5040a0..fa320324234e 100644 --- a/mlir/include/mlir-c/Dialect/Transform/Interpreter.h +++ b/mlir/include/mlir-c/Dialect/Transform/Interpreter.h @@ -60,7 +60,7 @@ MLIR_CAPI_EXPORTED void mlirTransformOptionsDestroy(MlirTransformOptions transformOptions); //----------------------------------------------------------------------------// -// Transform interpreter. +// Transform interpreter and utilities. //----------------------------------------------------------------------------// /// Applies the transformation script starting at the given transform root @@ -72,6 +72,16 @@ MLIR_CAPI_EXPORTED MlirLogicalResult mlirTransformApplyNamedSequence( MlirOperation payload, MlirOperation transformRoot, MlirOperation transformModule, MlirTransformOptions transformOptions); +/// Merge the symbols from `other` into `target`, potentially renaming them to +/// avoid conflicts. Private symbols may be renamed during the merge, public +/// symbols must have at most one declaration. A name conflict in public symbols +/// is reported as an error before returning a failure. +/// +/// Note that this clones the `other` operation unlike the C++ counterpart that +/// takes ownership. +MLIR_CAPI_EXPORTED MlirLogicalResult +mlirMergeSymbolsIntoFromClone(MlirOperation target, MlirOperation other); + #ifdef __cplusplus } #endif diff --git a/mlir/lib/Bindings/Python/TransformInterpreter.cpp b/mlir/lib/Bindings/Python/TransformInterpreter.cpp index 6517f8c39dfa..f6b4532b1b6b 100644 --- a/mlir/lib/Bindings/Python/TransformInterpreter.cpp +++ b/mlir/lib/Bindings/Python/TransformInterpreter.cpp @@ -82,6 +82,21 @@ static void populateTransformInterpreterSubmodule(py::module &m) { py::arg("payload_root"), py::arg("transform_root"), py::arg("transform_module"), py::arg("transform_options") = PyMlirTransformOptions()); + + m.def( + "copy_symbols_and_merge_into", + [](MlirOperation target, MlirOperation other) { + mlir::python::CollectDiagnosticsToStringScope scope( + mlirOperationGetContext(target)); + + MlirLogicalResult result = mlirMergeSymbolsIntoFromClone(target, other); + if (mlirLogicalResultIsFailure(result)) { + throw py::value_error( + "Failed to merge symbols.\nDiagnostic message " + + scope.takeMessage()); + } + }, + py::arg("target"), py::arg("other")); } PYBIND11_MODULE(_mlirTransformInterpreter, m) { diff --git a/mlir/lib/CAPI/Dialect/TransformInterpreter.cpp b/mlir/lib/CAPI/Dialect/TransformInterpreter.cpp index eb6951dc5584..145455e1c1b3 100644 --- a/mlir/lib/CAPI/Dialect/TransformInterpreter.cpp +++ b/mlir/lib/CAPI/Dialect/TransformInterpreter.cpp @@ -15,6 +15,7 @@ #include "mlir/CAPI/IR.h" #include "mlir/CAPI/Support.h" #include "mlir/CAPI/Wrap.h" +#include "mlir/Dialect/Transform/IR/Utils.h" #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/Dialect/Transform/Transforms/TransformInterpreterUtils.h" @@ -71,4 +72,12 @@ MlirLogicalResult mlirTransformApplyNamedSequence( unwrap(payload), unwrap(transformRoot), cast(unwrap(transformModule)), *unwrap(transformOptions))); } + +MlirLogicalResult mlirMergeSymbolsIntoFromClone(MlirOperation target, + MlirOperation other) { + OwningOpRef otherOwning(unwrap(other)->clone()); + LogicalResult result = transform::detail::mergeSymbolsInto( + unwrap(target), std::move(otherOwning)); + return wrap(result); +} } diff --git a/mlir/python/mlir/dialects/transform/interpreter/__init__.py b/mlir/python/mlir/dialects/transform/interpreter/__init__.py index 6145b99224eb..34cdc43cb617 100644 --- a/mlir/python/mlir/dialects/transform/interpreter/__init__.py +++ b/mlir/python/mlir/dialects/transform/interpreter/__init__.py @@ -5,7 +5,6 @@ from ....ir import Operation from ...._mlir_libs import _mlirTransformInterpreter as _cextTransformInterpreter - TransformOptions = _cextTransformInterpreter.TransformOptions @@ -31,3 +30,12 @@ def apply_named_sequence( _cextTransformInterpreter.apply_named_sequence(*args) else: _cextTransformInterpreter(*args, transform_options) + + +def copy_symbols_and_merge_into(target, other): + """Copies symbols from other into target, renaming private symbols to avoid + duplicates. Raises an error if copying would lead to duplicate public + symbols.""" + _cextTransformInterpreter.copy_symbols_and_merge_into( + _unpack_operation(target), _unpack_operation(other) + ) diff --git a/mlir/test/python/dialects/transform_interpreter.py b/mlir/test/python/dialects/transform_interpreter.py index 740c49f76a26..807a98c49327 100644 --- a/mlir/test/python/dialects/transform_interpreter.py +++ b/mlir/test/python/dialects/transform_interpreter.py @@ -54,3 +54,79 @@ def failed(): assert ( "must implement TransformOpInterface to be used as transform root" in str(e) ) + + +print_root_via_include_module = """ +module @print_root_via_include_module attributes {transform.with_named_sequence} { + transform.named_sequence private @callee1(%root: !transform.any_op {transform.readonly}) + transform.named_sequence private @callee2(%root: !transform.any_op {transform.readonly}) + transform.named_sequence @__transform_main(%root: !transform.any_op) { + transform.include @callee2 failures(propagate) + (%root) : (!transform.any_op) -> () + transform.yield + } +}""" + +callee2_definition = """ +module attributes {transform.with_named_sequence} { + transform.named_sequence private @callee1(%root: !transform.any_op {transform.readonly}) + transform.named_sequence @callee2(%root: !transform.any_op {transform.readonly}) { + transform.include @callee1 failures(propagate) + (%root) : (!transform.any_op) -> () + transform.yield + } +} +""" + +callee1_definition = """ +module attributes {transform.with_named_sequence} { + transform.named_sequence @callee1(%root: !transform.any_op {transform.readonly}) { + transform.print %root { name = \"from interpreter\" }: !transform.any_op + transform.yield + } +} +""" + + +@test_in_context +def include(): + main = ir.Module.parse(print_root_via_include_module) + callee1 = ir.Module.parse(callee1_definition) + callee2 = ir.Module.parse(callee2_definition) + interp.copy_symbols_and_merge_into(main, callee1) + interp.copy_symbols_and_merge_into(main, callee2) + + # CHECK: @print_root_via_include_module + # CHECK: transform.named_sequence @__transform_main + # CHECK: transform.include @callee2 + # + # CHECK: transform.named_sequence @callee1 + # CHECK: transform.print + # + # CHECK: transform.named_sequence @callee2 + # CHECK: transform.include @callee1 + interp.apply_named_sequence(main, main.body.operations[0], main) + + +@test_in_context +def partial_include(): + main = ir.Module.parse(print_root_via_include_module) + callee2 = ir.Module.parse(callee2_definition) + interp.copy_symbols_and_merge_into(main, callee2) + + try: + interp.apply_named_sequence(main, main.body.operations[0], main) + except ValueError as e: + assert "Failed to apply" in str(e) + + +@test_in_context +def repeated_include(): + main = ir.Module.parse(print_root_via_include_module) + callee2 = ir.Module.parse(callee2_definition) + interp.copy_symbols_and_merge_into(main, callee2) + + try: + interp.copy_symbols_and_merge_into(main, callee2) + except ValueError as e: + assert "doubly defined symbol @callee2" in str(e) -- GitLab From 20d653fdb2d4d6eafa4575cd954beaf7ecad113a Mon Sep 17 00:00:00 2001 From: Krzysztof Parzyszek Date: Wed, 17 Apr 2024 08:04:19 -0500 Subject: [PATCH 049/862] [LLVM][CodeGen] Fix register lane liveness tracking in RegisterPressure (#88892) Re-enable an old assertion in `decreaseSetPressure`. --- llvm/lib/CodeGen/RegisterPressure.cpp | 39 ++++++++++++++------------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/llvm/lib/CodeGen/RegisterPressure.cpp b/llvm/lib/CodeGen/RegisterPressure.cpp index f86aa3a16720..3fa22447f416 100644 --- a/llvm/lib/CodeGen/RegisterPressure.cpp +++ b/llvm/lib/CodeGen/RegisterPressure.cpp @@ -64,7 +64,7 @@ static void increaseSetPressure(std::vector &CurrSetPressure, static void decreaseSetPressure(std::vector &CurrSetPressure, const MachineRegisterInfo &MRI, Register Reg, LaneBitmask PrevMask, LaneBitmask NewMask) { - //assert((NewMask & !PrevMask) == 0 && "Must not add bits"); + assert((NewMask & ~PrevMask).none() && "Must not add bits"); if (NewMask.any() || PrevMask.none()) return; @@ -617,17 +617,11 @@ void RegisterOperands::adjustLaneLiveness(const LiveIntervals &LIS, ++I; } } - for (auto *I = Uses.begin(); I != Uses.end();) { - LaneBitmask LiveBefore = getLiveLanesAt(LIS, MRI, true, I->RegUnit, - Pos.getBaseIndex()); - LaneBitmask LaneMask = I->LaneMask & LiveBefore; - if (LaneMask.none()) { - I = Uses.erase(I); - } else { - I->LaneMask = LaneMask; - ++I; - } - } + + // For uses just copy the information from LIS. + for (auto &[RegUnit, LaneMask] : Uses) + LaneMask = getLiveLanesAt(LIS, MRI, true, RegUnit, Pos.getBaseIndex()); + if (AddFlagsMI != nullptr) { for (const RegisterMaskPair &P : DeadDefs) { Register RegUnit = P.RegUnit; @@ -1060,18 +1054,27 @@ void RegPressureTracker::bumpUpwardPressure(const MachineInstr *MI) { // Kill liveness at live defs. for (const RegisterMaskPair &P : RegOpers.Defs) { Register Reg = P.RegUnit; - LaneBitmask LiveLanes = LiveRegs.contains(Reg); + LaneBitmask LiveAfter = LiveRegs.contains(Reg); LaneBitmask UseLanes = getRegLanes(RegOpers.Uses, Reg); LaneBitmask DefLanes = P.LaneMask; - LaneBitmask LiveAfter = (LiveLanes & ~DefLanes) | UseLanes; - decreaseRegPressure(Reg, LiveLanes, LiveAfter); + LaneBitmask LiveBefore = (LiveAfter & ~DefLanes) | UseLanes; + + // There may be parts of the register that were dead before the + // instruction, but became live afterwards. Similarly, some parts + // may have been killed in this instruction. + decreaseRegPressure(Reg, LiveAfter, LiveAfter & LiveBefore); + increaseRegPressure(Reg, LiveAfter, ~LiveAfter & LiveBefore); } // Generate liveness for uses. for (const RegisterMaskPair &P : RegOpers.Uses) { Register Reg = P.RegUnit; - LaneBitmask LiveLanes = LiveRegs.contains(Reg); - LaneBitmask LiveAfter = LiveLanes | P.LaneMask; - increaseRegPressure(Reg, LiveLanes, LiveAfter); + // If this register was also in a def operand, we've handled it + // with defs. + if (getRegLanes(RegOpers.Defs, Reg).any()) + continue; + LaneBitmask LiveAfter = LiveRegs.contains(Reg); + LaneBitmask LiveBefore = LiveAfter | P.LaneMask; + increaseRegPressure(Reg, LiveAfter, LiveBefore); } } -- GitLab From 1fc72dbc807fb138cafd05501e2e31beaa574693 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Wed, 17 Apr 2024 21:10:56 +0800 Subject: [PATCH 050/862] [RISCV] Add test for doLocalPostpass issue not checking if VL was modified. NFC --- .../test/CodeGen/RISCV/rvv/vsetvli-insert.mir | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.mir b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.mir index 1850abe6363b..d9a87c2cb12a 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.mir +++ b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.mir @@ -76,6 +76,10 @@ ret void } + define void @postpass_modify_vl() { + ret void + } + declare @llvm.riscv.vadd.nxv1i64.nxv1i64.i64(, , , i64) #1 declare @llvm.riscv.vle.nxv1i64.i64(, ptr nocapture, i64) #4 @@ -503,3 +507,24 @@ body: | %5:vr = PseudoVMV_V_I_MF2 $noreg, 1, 2, 5, 0 PseudoRET ... +--- +name: postpass_modify_vl +tracksRegLiveness: true +body: | + bb.0: + liveins: $x1 + ; CHECK-LABEL: name: postpass_modify_vl + ; CHECK: liveins: $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: $x0 = PseudoVSETIVLI 3, 216 /* e64, m1, ta, ma */, implicit-def $vl, implicit-def $vtype + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $vtype + ; CHECK-NEXT: $vl = COPY $x1 + ; CHECK-NEXT: [[PseudoVADD_VV_M1_:%[0-9]+]]:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, 3, 6 /* e64 */, 0 /* tu, mu */, implicit $vl, implicit $vtype + ; CHECK-NEXT: PseudoRET + dead $x0 = PseudoVSETIVLI 3, 216, implicit-def $vl, implicit-def $vtype + %1:gpr = COPY $vtype + $vl = COPY $x1 + dead $x0 = PseudoVSETIVLI 3, 216, implicit-def $vl, implicit-def $vtype + %4:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, 3, 6, 0 + PseudoRET +... -- GitLab From 76ad2897480a85532eee93daf041246881772693 Mon Sep 17 00:00:00 2001 From: Zaara Syeda Date: Wed, 17 Apr 2024 09:24:53 -0400 Subject: [PATCH 051/862] [PowerPC] 32-bit large code-model support for toc-data (#85129) This patch adds the pseudo op ADDItocL for 32-bit large code-model support for toc-data. --- llvm/lib/Target/PowerPC/P10InstrResources.td | 2 +- llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp | 61 ++++++++++++------ llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp | 35 +++++++--- llvm/lib/Target/PowerPC/PPCInstrInfo.cpp | 1 + llvm/lib/Target/PowerPC/PPCInstrInfo.td | 4 +- llvm/lib/Target/PowerPC/PPCMacroFusion.def | 10 +-- llvm/test/CodeGen/PowerPC/toc-data.ll | 67 ++++++++++++++++++++ 7 files changed, 144 insertions(+), 36 deletions(-) diff --git a/llvm/lib/Target/PowerPC/P10InstrResources.td b/llvm/lib/Target/PowerPC/P10InstrResources.td index 5015ba887d0b..32cebb65cb56 100644 --- a/llvm/lib/Target/PowerPC/P10InstrResources.td +++ b/llvm/lib/Target/PowerPC/P10InstrResources.td @@ -881,7 +881,7 @@ def : InstRW<[P10W_FX_3C, P10W_DISP_ANY], // 3 Cycles ALU operations, 1 input operands def : InstRW<[P10W_FX_3C, P10W_DISP_ANY, P10FX_Read], (instrs - ADDI, ADDI8, ADDIdtprelL32, ADDItlsldLADDR32, ADDItocL8, LI, LI8, + ADDI, ADDI8, ADDIdtprelL32, ADDItlsldLADDR32, ADDItocL, ADDItocL8, LI, LI8, ADDIC, ADDIC8, ADDIS, ADDIS8, ADDISdtprelHA32, ADDIStocHA, ADDIStocHA8, LIS, LIS8, ADDME, ADDME8, diff --git a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp index 1c57b92057ff..6e1002c45d81 100644 --- a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp +++ b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp @@ -1148,15 +1148,27 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) { MCSymbolRefExpr::VariantKind VK = GetVKForMO(MO); - // Always use TOC on AIX. Map the global address operand to be a reference - // to the TOC entry we will synthesize later. 'TOCEntry' is a label used to - // reference the storage allocated in the TOC which contains the address of - // 'MOSymbol'. - MCSymbol *TOCEntry = - lookUpOrCreateTOCEntry(MOSymbol, getTOCEntryTypeForMO(MO), VK); - const MCExpr *Exp = MCSymbolRefExpr::create(TOCEntry, - MCSymbolRefExpr::VK_PPC_U, - OutContext); + // If the symbol isn't toc-data then use the TOC on AIX. + // Map the global address operand to be a reference to the TOC entry we + // will synthesize later. 'TOCEntry' is a label used to reference the + // storage allocated in the TOC which contains the address of 'MOSymbol'. + // If the toc-data attribute is used, the TOC entry contains the data + // rather than the address of the MOSymbol. + if (![](const MachineOperand &MO) { + if (!MO.isGlobal()) + return false; + + const GlobalVariable *GV = dyn_cast(MO.getGlobal()); + if (!GV) + return false; + + return GV->hasAttribute("toc-data"); + }(MO)) { + MOSymbol = lookUpOrCreateTOCEntry(MOSymbol, getTOCEntryTypeForMO(MO), VK); + } + + const MCExpr *Exp = MCSymbolRefExpr::create( + MOSymbol, MCSymbolRefExpr::VK_PPC_U, OutContext); TmpInst.getOperand(2) = MCOperand::createExpr(Exp); EmitToStreamer(*OutStreamer, TmpInst); return; @@ -1273,25 +1285,32 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) { EmitToStreamer(*OutStreamer, TmpInst); return; } + case PPC::ADDItocL: case PPC::ADDItocL8: { - // Transform %xd = ADDItocL8 %xs, @sym + // Transform %xd = ADDItocL %xs, @sym LowerPPCMachineInstrToMCInst(MI, TmpInst, *this); - // Change the opcode to ADDI8. If the global address is external, then - // generate a TOC entry and reference that. Otherwise, reference the - // symbol directly. - TmpInst.setOpcode(PPC::ADDI8); + unsigned Op = MI->getOpcode(); + + // Change the opcode to load address for tocdata + TmpInst.setOpcode(Op == PPC::ADDItocL8 ? PPC::ADDI8 : PPC::LA); const MachineOperand &MO = MI->getOperand(2); - assert((MO.isGlobal() || MO.isCPI()) && "Invalid operand for ADDItocL8."); + assert((Op == PPC::ADDItocL8) + ? (MO.isGlobal() || MO.isCPI()) + : MO.isGlobal() && "Invalid operand for ADDItocL8."); + assert(!(MO.isGlobal() && Subtarget->isGVIndirectSymbol(MO.getGlobal())) && + "Interposable definitions must use indirect accesses."); - LLVM_DEBUG(assert( - !(MO.isGlobal() && Subtarget->isGVIndirectSymbol(MO.getGlobal())) && - "Interposable definitions must use indirect access.")); + // Map the operand to its corresponding MCSymbol. + const MCSymbol *const MOSymbol = getMCSymbolForTOCPseudoMO(MO, *this); + + const MCExpr *Exp = MCSymbolRefExpr::create( + MOSymbol, + Op == PPC::ADDItocL8 ? MCSymbolRefExpr::VK_PPC_TOC_LO + : MCSymbolRefExpr::VK_PPC_L, + OutContext); - const MCExpr *Exp = - MCSymbolRefExpr::create(getMCSymbolForTOCPseudoMO(MO, *this), - MCSymbolRefExpr::VK_PPC_TOC_LO, OutContext); TmpInst.getOperand(2) = MCOperand::createExpr(Exp); EmitToStreamer(*OutStreamer, TmpInst); return; diff --git a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp index af82b6cdb180..2f647daa4bcb 100644 --- a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp +++ b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp @@ -510,7 +510,7 @@ SDNode *PPCDAGToDAGISel::getGlobalBaseReg() { } // Check if a SDValue has the toc-data attribute. -static bool hasTocDataAttr(SDValue Val, unsigned PointerSize) { +static bool hasTocDataAttr(SDValue Val) { GlobalAddressSDNode *GA = dyn_cast(Val); if (!GA) return false; @@ -6115,8 +6115,7 @@ void PPCDAGToDAGISel::Select(SDNode *N) { assert(isAIXABI && "ELF ABI already handled"); - if (hasTocDataAttr(N->getOperand(0), - CurDAG->getDataLayout().getPointerSize())) { + if (hasTocDataAttr(N->getOperand(0))) { replaceWith(PPC::ADDItoc, N, MVT::i32); return; } @@ -6128,8 +6127,7 @@ void PPCDAGToDAGISel::Select(SDNode *N) { if (isPPC64 && CModel == CodeModel::Small) { assert(isAIXABI && "ELF ABI handled in common SelectCode"); - if (hasTocDataAttr(N->getOperand(0), - CurDAG->getDataLayout().getPointerSize())) { + if (hasTocDataAttr(N->getOperand(0))) { replaceWith(PPC::ADDItoc8, N, MVT::i64); return; } @@ -6144,9 +6142,10 @@ void PPCDAGToDAGISel::Select(SDNode *N) { " ELF/AIX or 32-bit AIX in the following."); // Transforms the ISD::TOC_ENTRY node for 32-bit AIX large code model mode - // or 64-bit medium (ELF-only) or large (ELF and AIX) code model code. We - // generate two instructions as described below. The first source operand - // is a symbol reference. If it must be toc-referenced according to + // or 64-bit medium (ELF-only) or large (ELF and AIX) code model code non + // toc-data symbols. + // We generate two instructions as described below. The first source + // operand is a symbol reference. If it must be toc-referenced according to // Subtarget, we generate: // [32-bit AIX] // LWZtocL(@sym, ADDIStocHA(%r2, @sym)) @@ -6154,6 +6153,13 @@ void PPCDAGToDAGISel::Select(SDNode *N) { // LDtocL(@sym, ADDIStocHA8(%x2, @sym)) // Otherwise we generate: // ADDItocL8(ADDIStocHA8(%x2, @sym), @sym) + + // For large code model toc-data symbols we generate: + // [32-bit AIX] + // ADDItocL(ADDIStocHA(%x2, @sym), @sym) + // [64-bit AIX] + // Currently not supported. + SDValue GA = N->getOperand(0); SDValue TOCbase = N->getOperand(1); @@ -6161,6 +6167,19 @@ void PPCDAGToDAGISel::Select(SDNode *N) { SDNode *Tmp = CurDAG->getMachineNode( isPPC64 ? PPC::ADDIStocHA8 : PPC::ADDIStocHA, dl, VT, TOCbase, GA); + // On AIX if the symbol has the toc-data attribute it will be defined + // in the TOC entry, so we use an ADDItocL similar to the medium code + // model ELF abi. + if (isAIXABI && hasTocDataAttr(GA)) { + if (isPPC64) + report_fatal_error( + "64-bit large code model toc-data not yet supported"); + + ReplaceNode(N, CurDAG->getMachineNode(PPC::ADDItocL, dl, VT, + SDValue(Tmp, 0), GA)); + return; + } + if (PPCLowering->isAccessedAsGotIndirect(GA)) { // If it is accessed as got-indirect, we need an extra LWZ/LD to load // the address. diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp index 93874d65531a..b32f178ca38e 100644 --- a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp +++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp @@ -1090,6 +1090,7 @@ bool PPCInstrInfo::isReallyTriviallyReMaterializable( case PPC::LIS8: case PPC::ADDIStocHA: case PPC::ADDIStocHA8: + case PPC::ADDItocL: case PPC::ADDItocL8: case PPC::LOAD_STACK_GUARD: case PPC::PPCLdFixedAddr: diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/llvm/lib/Target/PowerPC/PPCInstrInfo.td index 6423e692d88c..43e3902cf407 100644 --- a/llvm/lib/Target/PowerPC/PPCInstrInfo.td +++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.td @@ -3346,11 +3346,13 @@ def ADDIStocHA : PPCEmitTimePseudo<(outs gprc:$rD), (ins gprc_nor0:$reg, tocentr "#ADDIStocHA", [(set i32:$rD, (PPCtoc_entry i32:$reg, tglobaladdr:$disp))]>; -// Local Data Transform +// TOC Data Transform AIX def ADDItoc : PPCEmitTimePseudo<(outs gprc:$rD), (ins tocentry32:$disp, gprc:$reg), "#ADDItoc", [(set i32:$rD, (PPCtoc_entry tglobaladdr:$disp, i32:$reg))]>; +def ADDItocL : PPCEmitTimePseudo<(outs gprc:$rD), (ins gprc_nor0:$reg, tocentry32:$disp), + "#ADDItocL", []>; // Get Global (GOT) Base Register offset, from the word immediately preceding // the function label. diff --git a/llvm/lib/Target/PowerPC/PPCMacroFusion.def b/llvm/lib/Target/PowerPC/PPCMacroFusion.def index fb6e656edb8b..1a61ae23e5d7 100644 --- a/llvm/lib/Target/PowerPC/PPCMacroFusion.def +++ b/llvm/lib/Target/PowerPC/PPCMacroFusion.def @@ -32,7 +32,7 @@ // {addi} followed by one of these {lxvd2x, lxvw4x, lxvdsx, lvebx, lvehx, // lvewx, lvx, lxsdx} FUSION_FEATURE(AddiLoad, hasAddiLoadFusion, 2, \ - FUSION_OP_SET(ADDI, ADDI8, ADDItocL8), \ + FUSION_OP_SET(ADDI, ADDI8, ADDItocL, ADDItocL8), \ FUSION_OP_SET(LXVD2X, LXVW4X, LXVDSX, LVEBX, LVEHX, LVEWX, \ LVX, LXSDX)) @@ -134,13 +134,13 @@ FUSION_FEATURE(XorisXori, hasWideImmFusion, 1, FUSION_OP_SET(XORIS, XORIS8), // addis rx,ra,si - addi rt,rx,SI, SI >= 0 FUSION_FEATURE(AddisAddi, hasWideImmFusion, 1, - FUSION_OP_SET(ADDIS, ADDIS8, ADDIStocHA8), - FUSION_OP_SET(ADDI, ADDI8, ADDItocL8)) + FUSION_OP_SET(ADDIS, ADDIS8, ADDIStocHA8, ADDIStocHA), + FUSION_OP_SET(ADDI, ADDI8, ADDItocL8, ADDItocL)) // addi rx,ra,si - addis rt,rx,SI, ra > 0, SI >= 2 FUSION_FEATURE(AddiAddis, hasWideImmFusion, 1, - FUSION_OP_SET(ADDI, ADDI8, ADDItocL8), - FUSION_OP_SET(ADDIS, ADDIS8, ADDIStocHA8)) + FUSION_OP_SET(ADDI, ADDI8, ADDItocL8, ADDItocL), + FUSION_OP_SET(ADDIS, ADDIS8, ADDIStocHA8, ADDIStocHA)) // mtctr - { bcctr,bcctrl } FUSION_FEATURE(ZeroMoveCTR, hasZeroMoveFusion, -1, diff --git a/llvm/test/CodeGen/PowerPC/toc-data.ll b/llvm/test/CodeGen/PowerPC/toc-data.ll index cbf3be9fcaad..7f7afe76cfcd 100644 --- a/llvm/test/CodeGen/PowerPC/toc-data.ll +++ b/llvm/test/CodeGen/PowerPC/toc-data.ll @@ -12,6 +12,14 @@ ; RUN: llc -mtriple powerpc-ibm-aix-xcoff -verify-machineinstrs -O0 < %s | FileCheck %s --check-prefix TEST32 ; RUN: llc -mtriple powerpc64-ibm-aix-xcoff -verify-machineinstrs -O0 < %s | FileCheck %s --check-prefix TEST64 +; RUN: llc -mtriple powerpc-ibm-aix-xcoff -code-model=large -verify-machineinstrs < %s \ +; RUN: -stop-before=ppc-vsx-copy | FileCheck %s --check-prefix CHECK32LARGE +; RUN: llc -mtriple powerpc-ibm-aix-xcoff -code-model=large -verify-machineinstrs < %s | FileCheck %s --check-prefix TEST32LARGE + +; Global variables i and f have the toc-data attribute. +; In the following functions, those writing to or reading from +; variables i and f should use the toc-data access pattern. +; All remaining variables should use the regular toc access sequence. @i = dso_local global i32 0, align 4 #0 @d = dso_local local_unnamed_addr global double 3.141590e+00, align 8 @f = dso_local local_unnamed_addr global float 0x4005BE76C0000000, align 4 #0 @@ -44,6 +52,16 @@ define dso_local void @write_int(i32 signext %in) { ; TEST64: la 4, i[TD](2) ; TEST64-NEXT: stw 3, 0(4) +; CHECK32LARGE: name: write_int +; CHECK32LARGE: %[[SCRATCH1:[0-9]+]]:gprc_and_gprc_nor0 = ADDIStocHA $r2, @i +; CHECK32LARGE-NEXT: %[[SCRATCH2:[0-9]+]]:gprc_and_gprc_nor0 = ADDItocL killed %[[SCRATCH1]], @i +; CHECK32LARGE-NEXT: STW %{{[0-9]+}}, 0, killed %[[SCRATCH2]] :: (store (s32) into @i) + +; FIXME: peephole optimization opportunity for lower part relocation @l to the consuming stw +; TEST32LARGE: .write_int: +; TEST32LARGE: addis 4, i[TD]@u(2) +; TEST32LARGE-NEXT: la 4, i[TD]@l(4) +; TEST32LARGE-NEXT: stw 3, 0(4) define dso_local i64 @read_ll() { entry: @@ -70,6 +88,15 @@ define dso_local i64 @read_ll() { ; TEST64: ld 3, L..C0(2) ; TEST64-NEXT: ld 3, 0(3) +; CHECK32LARGE: name: read_ll +; CHECK32LARGE: %[[SCRATCH1:[0-9]+]]:gprc_and_gprc_nor0 = ADDIStocHA $r2, @ll +; CHECK32LARGE: LWZtocL @ll, killed %[[SCRATCH1]] :: (load (s32) from got) + +; TEST32LARGE: .read_ll: +; TEST32LARGE: addis 3, L..C0@u(2) +; TEST32LARGE-NEXT: lwz 4, L..C0@l(3) +; TEST32LARGE-NEXT: lwz 3, 0(4) +; TEST32LARGE-NEXT: lwz 4, 4(4) define dso_local float @read_float() { entry: @@ -96,6 +123,16 @@ define dso_local float @read_float() { ; TEST64: la 3, f[TD](2) ; TEST64-NEXT: lfs 1, 0(3) +; CHECK32LARGE: name: read_float +; CHECK32LARGE: %[[SCRATCH1:[0-9]+]]:gprc_and_gprc_nor0 = ADDIStocHA $r2, @f +; CHECK32LARGE-NEXT: %[[SCRATCH2:[0-9]+]]:gprc_and_gprc_nor0 = ADDItocL killed %[[SCRATCH1]], @f +; CHECK32LARGE-NEXT: LFS 0, killed %[[SCRATCH2]] :: (dereferenceable load (s32) from @f) + +; FIXME: peephole optimization opportunity for lower part relocation @l to the consuming lfs +; TEST32LARGE: .read_float: +; TEST32LARGE: addis 3, f[TD]@u(2) +; TEST32LARGE-NEXT: la 3, f[TD]@l(3) +; TEST32LARGE-NEXT: lfs 1, 0(3) define dso_local void @write_double(double %in) { entry: @@ -121,6 +158,14 @@ define dso_local void @write_double(double %in) { ; TEST64: ld 3, L..C1(2) ; TEST64-NEXT: stfd 1, 0(3) +; CHECK32LARGE: name: write_double +; CHECK32LARGE: %[[SCRATCH1:[0-9]+]]:gprc_and_gprc_nor0 = ADDIStocHA $r2, @d +; CHECK32LARGE: LWZtocL @d, killed %[[SCRATCH1]] :: (load (s32) from got) + +; TEST32LARGE: .write_double: +; TEST32LARGE: addis 3, L..C1@u(2) +; TEST32LARGE-NEXT: lwz 3, L..C1@l(3) +; TEST32LARGE-NEXT: stfd 1, 0(3) define dso_local nonnull ptr @addr() { entry: @@ -144,6 +189,15 @@ define dso_local nonnull ptr @addr() { ; TEST64: .addr ; TEST64: la 3, i[TD](2) +; CHECK32LARGE: name: addr +; CHECK32LARGE: %[[SCRATCH1:[0-9]+]]:gprc_and_gprc_nor0 = ADDIStocHA $r2, @i +; CHECK32LARGE-NEXT: %[[SCRATCH2:[0-9]+]]:gprc = ADDItocL killed %[[SCRATCH1]], @i +; CHECK32LARGE-NEXT: $r3 = COPY %[[SCRATCH2]] + +; TEST32LARGE: .addr: +; TEST32LARGE: addis 3, i[TD]@u(2) +; TEST32LARGE-NEXT: la 3, i[TD]@l(3) + ; TEST32: .toc ; TEST32: .tc ll[TC],ll[RW] ; TEST32-NOT: .csect ll[TD] @@ -170,4 +224,17 @@ define dso_local nonnull ptr @addr() { ; TEST64-NEXT: .globl f[TD] ; TEST64-NOT: .tc f[TD],f[RW] +; TEST32LARGE: .toc +; TEST32LARGE: .tc ll[TE],ll[RW] +; TEST32LARGE-NOT: .csect ll[TD] +; TEST32LARGE: .tc d[TE],d[RW] +; TEST32LARGE-NOT: .csect d[TD],2 +; TEST32LARGE: .csect i[TD],2 +; TEST32LARGE-NEXT: .globl i[TD] +; TEST32LARGE-NEXT: .align 2 +; TEST32LARGE-NOT: .tc i[TE],i[RW] +; TEST32LARGE: .csect f[TD],2 +; TEST32LARGE-NEXT: .globl f[TD] +; TEST32LARGE-NOT: .tc f[TE],f[RW] + attributes #0 = { "toc-data" } -- GitLab From edbeae373489a2e710f328ceba50b4740c738217 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Wed, 17 Apr 2024 21:12:08 +0800 Subject: [PATCH 052/862] [RISCV] Explicitly bail if something modifies VL/VTYPE in doLocalPostpass If an instruction between MI and NextMI uses VL or VTYPE we demand the respective fields so as to not clobber them at their uses. But we don't consider if something might modify VL or VTYPE, and will happily coalesce two vsetvlis when we need to preserve them. This fixes this by skipping to the next vsetvli. Demanding the fields isn't enough, as we need to preserve the VL and VTYPE values even if no fields are demanded. In practice this doesn't happen, presumably due to there not being any instructions that write to VL or VTYPE without reading them. But I noticed this whilst working on a separate patch and split it out. --- llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp | 3 +++ llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.mir | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp index aab91adbb64b..fa37d1ccccd7 100644 --- a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp +++ b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp @@ -1538,6 +1538,9 @@ void RISCVInsertVSETVLI::doLocalPostpass(MachineBasicBlock &MBB) { if (!isVectorConfigInstr(MI)) { doUnion(Used, getDemanded(MI, MRI, ST)); + if (MI.isCall() || MI.isInlineAsm() || MI.modifiesRegister(RISCV::VL) || + MI.modifiesRegister(RISCV::VTYPE)) + NextMI = nullptr; continue; } diff --git a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.mir b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.mir index d9a87c2cb12a..e8620c848f8d 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.mir +++ b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.mir @@ -516,9 +516,10 @@ body: | ; CHECK-LABEL: name: postpass_modify_vl ; CHECK: liveins: $x1 ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: $x0 = PseudoVSETIVLI 3, 216 /* e64, m1, ta, ma */, implicit-def $vl, implicit-def $vtype + ; CHECK-NEXT: dead $x0 = PseudoVSETIVLI 3, 216 /* e64, m1, ta, ma */, implicit-def $vl, implicit-def $vtype ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $vtype ; CHECK-NEXT: $vl = COPY $x1 + ; CHECK-NEXT: dead $x0 = PseudoVSETIVLI 3, 216 /* e64, m1, ta, ma */, implicit-def $vl, implicit-def $vtype ; CHECK-NEXT: [[PseudoVADD_VV_M1_:%[0-9]+]]:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, 3, 6 /* e64 */, 0 /* tu, mu */, implicit $vl, implicit $vtype ; CHECK-NEXT: PseudoRET dead $x0 = PseudoVSETIVLI 3, 216, implicit-def $vl, implicit-def $vtype -- GitLab From 38205717501237f2b7a57eaabe65a8367e5f91c3 Mon Sep 17 00:00:00 2001 From: Aaron Ballman Date: Tue, 16 Apr 2024 14:57:27 -0400 Subject: [PATCH 053/862] [C99] Remove WG14 N522 from the C status page This paper is about type compatibility rules that changed in C99, but this is only applicable across translation units and so there's nothing for us to test. The specific change was that C89 allowed different tag types (e.g., struct and union) to be compatible and C99 tightened that restriction. This is a case where the user gets whatever they get if they link two TUs with incompatible tag types. --- clang/www/c_status.html | 5 ----- 1 file changed, 5 deletions(-) diff --git a/clang/www/c_status.html b/clang/www/c_status.html index 9893170ae847..dfc1afefda24 100644 --- a/clang/www/c_status.html +++ b/clang/www/c_status.html @@ -295,11 +295,6 @@ conformance.

N570 Yes - - new structure type compatibility (tag compatibility) - N522 - Unknown - additional predefined macro names Unknown -- GitLab From 41b7341d6b27adf81262a5a0bd4e430675b73bbb Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Wed, 17 Apr 2024 14:56:46 +0100 Subject: [PATCH 054/862] [VPlan] Factor out helper to recursively collect all users (NFCI). Factor out logic to collect all users recursively to be re-used in https://github.com/llvm/llvm-project/pull/87816. --- llvm/lib/Transforms/Vectorize/VPlan.h | 5 ++++ .../Transforms/Vectorize/VPlanTransforms.cpp | 30 +++++++++---------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 148227f1f1a5..334b10e2e5d0 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1049,6 +1049,11 @@ public: R->getVPDefID() == VPRecipeBase::VPVectorPointerSC; } + static inline bool classof(const VPUser *U) { + auto *R = dyn_cast(U); + return R && classof(R); + } + /// Drop all poison-generating flags. void dropPoisonGeneratingFlags() { // NOTE: This needs to be kept in-sync with diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 382bf5ac1140..78d0b5b95c5e 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -852,6 +852,18 @@ bool VPlanTransforms::adjustFixedOrderRecurrences(VPlan &Plan, return true; } +static SmallVector collectUsersRecursively(VPValue *V) { + SetVector Users(V->user_begin(), V->user_end()); + for (unsigned I = 0; I != Users.size(); ++I) { + VPRecipeBase *Cur = dyn_cast(Users[I]); + if (!Cur || isa(Cur)) + continue; + for (VPValue *V : Cur->definedValues()) + Users.insert(V->user_begin(), V->user_end()); + } + return Users.takeVector(); +} + void VPlanTransforms::clearReductionWrapFlags(VPlan &Plan) { for (VPRecipeBase &R : Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()) { @@ -863,24 +875,10 @@ void VPlanTransforms::clearReductionWrapFlags(VPlan &Plan) { if (RK != RecurKind::Add && RK != RecurKind::Mul) continue; - SmallSetVector Worklist; - Worklist.insert(PhiR); - - for (unsigned I = 0; I != Worklist.size(); ++I) { - VPValue *Cur = Worklist[I]; - if (auto *RecWithFlags = - dyn_cast(Cur->getDefiningRecipe())) { + for (VPUser *U : collectUsersRecursively(PhiR)) + if (auto *RecWithFlags = dyn_cast(U)) { RecWithFlags->dropPoisonGeneratingFlags(); } - - for (VPUser *U : Cur->users()) { - auto *UserRecipe = dyn_cast(U); - if (!UserRecipe) - continue; - for (VPValue *V : UserRecipe->definedValues()) - Worklist.insert(V); - } - } } } -- GitLab From 856d1c44103f09f2ed0448001de9dcda63055733 Mon Sep 17 00:00:00 2001 From: Jay Foad Date: Wed, 17 Apr 2024 14:58:13 +0100 Subject: [PATCH 055/862] [AMDGPU] Fix predicates for BUFFER_ATOMIC_FMIN/FMAX patterns (#89066) Use OtherPredicates to avoid interfering with other uses of SubtargetPredicate for GFX12. --- llvm/lib/Target/AMDGPU/BUFInstructions.td | 2 +- .../AMDGPU/fp-min-max-buffer-atomics.ll | 72 +++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td index 273f92abf354..8053d89aeb0a 100644 --- a/llvm/lib/Target/AMDGPU/BUFInstructions.td +++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td @@ -1726,7 +1726,7 @@ let SubtargetPredicate = isGFX12Plus in { defm : SIBufferAtomicPat_Common<"SIbuffer_atomic_cond_sub_u32", i32, "BUFFER_ATOMIC_COND_SUB_U32_VBUFFER", ["noret"]>; } -let SubtargetPredicate = isGFX6GFX7GFX10Plus in { +let OtherPredicates = [isGFX6GFX7GFX10Plus] in { defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f32, "BUFFER_ATOMIC_FMIN">; defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f32, "BUFFER_ATOMIC_FMAX">; } diff --git a/llvm/test/CodeGen/AMDGPU/fp-min-max-buffer-atomics.ll b/llvm/test/CodeGen/AMDGPU/fp-min-max-buffer-atomics.ll index 0c62b52eb92a..587340c7aa34 100644 --- a/llvm/test/CodeGen/AMDGPU/fp-min-max-buffer-atomics.ll +++ b/llvm/test/CodeGen/AMDGPU/fp-min-max-buffer-atomics.ll @@ -4,12 +4,14 @@ ; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs | FileCheck %s -check-prefix=GFX10 ; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1030 -verify-machineinstrs | FileCheck %s -check-prefix=GFX1030 ; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs | FileCheck %s -check-prefix=GFX1100 +; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs | FileCheck %s -check-prefix=GFX12 ; RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck %s -check-prefix=G_SI ; RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=hawaii -verify-machineinstrs | FileCheck %s -check-prefix=G_GFX7 ; RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs | FileCheck %s -check-prefix=G_GFX10 ; RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=gfx1030 -verify-machineinstrs | FileCheck %s -check-prefix=G_GFX1030 ; RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs | FileCheck %s -check-prefix=G_GFX1100 +; RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs | FileCheck %s -check-prefix=GFX12 declare float @llvm.amdgcn.raw.buffer.atomic.fmin.f32(float, <4 x i32>, i32, i32, i32 immarg) declare float @llvm.amdgcn.raw.buffer.atomic.fmax.f32(float, <4 x i32>, i32, i32, i32 immarg) @@ -70,6 +72,18 @@ define amdgpu_kernel void @raw_buffer_atomic_min_noret_f32(<4 x i32> inreg %rsrc ; GFX1100-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX1100-NEXT: s_endpgm ; +; GFX12-LABEL: raw_buffer_atomic_min_noret_f32: +; GFX12: ; %bb.0: ; %main_body +; GFX12-NEXT: s_clause 0x1 +; GFX12-NEXT: s_load_b64 s[4:5], s[0:1], 0x34 +; GFX12-NEXT: s_load_b128 s[0:3], s[0:1], 0x24 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5 +; GFX12-NEXT: buffer_atomic_min_num_f32 v0, v1, s[0:3], null offen +; GFX12-NEXT: s_nop 0 +; GFX12-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX12-NEXT: s_endpgm +; ; G_SI-LABEL: raw_buffer_atomic_min_noret_f32: ; G_SI: ; %bb.0: ; %main_body ; G_SI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0xd @@ -170,6 +184,15 @@ define amdgpu_ps void @raw_buffer_atomic_min_rtn_f32(<4 x i32> inreg %rsrc, floa ; GFX1100-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX1100-NEXT: s_endpgm ; +; GFX12-LABEL: raw_buffer_atomic_min_rtn_f32: +; GFX12: ; %bb.0: ; %main_body +; GFX12-NEXT: buffer_atomic_min_num_f32 v0, v1, s[0:3], null offen th:TH_ATOMIC_RETURN +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_store_b32 v[0:1], v0, off +; GFX12-NEXT: s_nop 0 +; GFX12-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX12-NEXT: s_endpgm +; ; G_SI-LABEL: raw_buffer_atomic_min_rtn_f32: ; G_SI: ; %bb.0: ; %main_body ; G_SI-NEXT: buffer_atomic_fmin v0, v1, s[0:3], 0 offen glc @@ -292,6 +315,20 @@ define amdgpu_kernel void @raw_buffer_atomic_min_rtn_f32_off4_slc(<4 x i32> inre ; GFX1100-NEXT: ds_store_b32 v1, v0 ; GFX1100-NEXT: s_endpgm ; +; GFX12-LABEL: raw_buffer_atomic_min_rtn_f32_off4_slc: +; GFX12: ; %bb.0: ; %main_body +; GFX12-NEXT: s_clause 0x1 +; GFX12-NEXT: s_load_b96 s[4:6], s[0:1], 0x34 +; GFX12-NEXT: s_load_b128 s[0:3], s[0:1], 0x24 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5 +; GFX12-NEXT: s_mov_b32 s4, 4 +; GFX12-NEXT: buffer_atomic_min_num_f32 v0, v1, s[0:3], s4 offen th:TH_ATOMIC_NT_RETURN +; GFX12-NEXT: v_mov_b32_e32 v1, s6 +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: ds_store_b32 v1, v0 +; GFX12-NEXT: s_endpgm +; ; G_SI-LABEL: raw_buffer_atomic_min_rtn_f32_off4_slc: ; G_SI: ; %bb.0: ; %main_body ; G_SI-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0xd @@ -427,6 +464,18 @@ define amdgpu_kernel void @raw_buffer_atomic_max_noret_f32(<4 x i32> inreg %rsrc ; GFX1100-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX1100-NEXT: s_endpgm ; +; GFX12-LABEL: raw_buffer_atomic_max_noret_f32: +; GFX12: ; %bb.0: ; %main_body +; GFX12-NEXT: s_clause 0x1 +; GFX12-NEXT: s_load_b64 s[4:5], s[0:1], 0x34 +; GFX12-NEXT: s_load_b128 s[0:3], s[0:1], 0x24 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5 +; GFX12-NEXT: buffer_atomic_max_num_f32 v0, v1, s[0:3], null offen +; GFX12-NEXT: s_nop 0 +; GFX12-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX12-NEXT: s_endpgm +; ; G_SI-LABEL: raw_buffer_atomic_max_noret_f32: ; G_SI: ; %bb.0: ; %main_body ; G_SI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0xd @@ -527,6 +576,15 @@ define amdgpu_ps void @raw_buffer_atomic_max_rtn_f32(<4 x i32> inreg %rsrc, floa ; GFX1100-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX1100-NEXT: s_endpgm ; +; GFX12-LABEL: raw_buffer_atomic_max_rtn_f32: +; GFX12: ; %bb.0: ; %main_body +; GFX12-NEXT: buffer_atomic_max_num_f32 v0, v1, s[0:3], null offen th:TH_ATOMIC_RETURN +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_store_b32 v[0:1], v0, off +; GFX12-NEXT: s_nop 0 +; GFX12-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX12-NEXT: s_endpgm +; ; G_SI-LABEL: raw_buffer_atomic_max_rtn_f32: ; G_SI: ; %bb.0: ; %main_body ; G_SI-NEXT: buffer_atomic_fmax v0, v1, s[0:3], 0 offen glc @@ -641,6 +699,20 @@ define amdgpu_kernel void @raw_buffer_atomic_max_rtn_f32_off4_slc(<4 x i32> inre ; GFX1100-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX1100-NEXT: s_endpgm ; +; GFX12-LABEL: raw_buffer_atomic_max_rtn_f32_off4_slc: +; GFX12: ; %bb.0: ; %main_body +; GFX12-NEXT: s_load_b256 s[0:7], s[0:1], 0x24 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5 +; GFX12-NEXT: s_mov_b32 s4, 4 +; GFX12-NEXT: buffer_atomic_max_num_f32 v0, v1, s[0:3], s4 offen th:TH_ATOMIC_NT_RETURN +; GFX12-NEXT: v_mov_b32_e32 v1, 0 +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_store_b32 v1, v0, s[6:7] +; GFX12-NEXT: s_nop 0 +; GFX12-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX12-NEXT: s_endpgm +; ; G_SI-LABEL: raw_buffer_atomic_max_rtn_f32_off4_slc: ; G_SI: ; %bb.0: ; %main_body ; G_SI-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x9 -- GitLab From 4f88c2311130791cf69da34b743b1b3ba7584a7b Mon Sep 17 00:00:00 2001 From: Guray Ozen Date: Wed, 17 Apr 2024 15:59:18 +0200 Subject: [PATCH 056/862] [mlir][py] Add NVGPU's `TensorMapDescriptorType` in py bindings (#88855) This PR adds NVGPU dialects' TensorMapDescriptorType in the py bindings. This is a follow-up issue from [this PR](https://github.com/llvm/llvm-project/pull/87153#discussion_r1546193095) --- mlir/include/mlir-c/Dialect/NVGPU.h | 11 ++++++ mlir/lib/Bindings/Python/DialectNVGPU.cpp | 41 +++++++++++++++++++++++ mlir/lib/CAPI/Dialect/NVGPU.cpp | 18 ++++++++++ mlir/python/CMakeLists.txt | 13 +++++++ mlir/python/mlir/dialects/nvgpu.py | 1 + mlir/test/python/dialects/nvgpu.py | 17 ++++++++++ 6 files changed, 101 insertions(+) create mode 100644 mlir/lib/Bindings/Python/DialectNVGPU.cpp diff --git a/mlir/include/mlir-c/Dialect/NVGPU.h b/mlir/include/mlir-c/Dialect/NVGPU.h index 580d566794c0..e58015a4a342 100644 --- a/mlir/include/mlir-c/Dialect/NVGPU.h +++ b/mlir/include/mlir-c/Dialect/NVGPU.h @@ -11,6 +11,7 @@ #define MLIR_C_DIALECT_NVGPU_H #include "mlir-c/IR.h" +#include "mlir-c/Support.h" #ifdef __cplusplus extern "C" { @@ -18,6 +19,16 @@ extern "C" { MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(NVGPU, nvgpu); +//===---------------------------------------------------------------------===// +// TensorMapDescriptorType +//===---------------------------------------------------------------------===// + +MLIR_CAPI_EXPORTED bool mlirTypeIsANVGPUTensorMapDescriptorType(MlirType type); + +MLIR_CAPI_EXPORTED MlirType mlirNVGPUTensorMapDescriptorTypeGet( + MlirContext ctx, MlirType tensorMemrefType, int swizzle, int l2promo, + int oobFill, int interleave); + #ifdef __cplusplus } #endif diff --git a/mlir/lib/Bindings/Python/DialectNVGPU.cpp b/mlir/lib/Bindings/Python/DialectNVGPU.cpp new file mode 100644 index 000000000000..341e4d55bcf2 --- /dev/null +++ b/mlir/lib/Bindings/Python/DialectNVGPU.cpp @@ -0,0 +1,41 @@ +//===--- DialectNvgpu.cpp - Pybind module for Nvgpu dialect API support ---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "mlir-c/Dialect/NVGPU.h" +#include "mlir-c/IR.h" +#include "mlir/Bindings/Python/PybindAdaptors.h" +#include + +namespace py = pybind11; +using namespace llvm; +using namespace mlir; +using namespace mlir::python; +using namespace mlir::python::adaptors; + +static void populateDialectNvgpuSubmodule(const pybind11::module &m) { + auto nvgpuTensorMapDescriptorType = mlir_type_subclass( + m, "TensorMapDescriptorType", mlirTypeIsANVGPUTensorMapDescriptorType); + + nvgpuTensorMapDescriptorType.def_classmethod( + "get", + [](py::object cls, MlirType tensorMemrefType, int swizzle, int l2promo, + int oobFill, int interleave, MlirContext ctx) { + return cls(mlirNVGPUTensorMapDescriptorTypeGet( + ctx, tensorMemrefType, swizzle, l2promo, oobFill, interleave)); + }, + "Gets an instance of TensorMapDescriptorType in the same context", + py::arg("cls"), py::arg("tensor_type"), py::arg("swizzle"), + py::arg("l2promo"), py::arg("oob_fill"), py::arg("interleave"), + py::arg("ctx") = py::none()); +} + +PYBIND11_MODULE(_mlirDialectsNvgpu, m) { + m.doc() = "MLIR NVGPU dialect."; + + populateDialectNvgpuSubmodule(m); +} diff --git a/mlir/lib/CAPI/Dialect/NVGPU.cpp b/mlir/lib/CAPI/Dialect/NVGPU.cpp index 02d10954a037..e6da529e1b6b 100644 --- a/mlir/lib/CAPI/Dialect/NVGPU.cpp +++ b/mlir/lib/CAPI/Dialect/NVGPU.cpp @@ -9,5 +9,23 @@ #include "mlir-c/Dialect/NVGPU.h" #include "mlir/CAPI/Registration.h" #include "mlir/Dialect/NVGPU/IR/NVGPUDialect.h" +#include "mlir/IR/BuiltinTypes.h" + +using namespace mlir; +using namespace mlir::nvgpu; MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(NVGPU, nvgpu, mlir::nvgpu::NVGPUDialect) + +bool mlirTypeIsANVGPUTensorMapDescriptorType(MlirType type) { + return isa(unwrap(type)); +} + +MlirType mlirNVGPUTensorMapDescriptorTypeGet(MlirContext ctx, + MlirType tensorMemrefType, + int swizzle, int l2promo, + int oobFill, int interleave) { + return wrap(nvgpu::TensorMapDescriptorType::get( + unwrap(ctx), cast(unwrap(tensorMemrefType)), + TensorMapSwizzleKind(swizzle), TensorMapL2PromoKind(l2promo), + TensorMapOOBKind(oobFill), TensorMapInterleaveKind(interleave))); +} diff --git a/mlir/python/CMakeLists.txt b/mlir/python/CMakeLists.txt index c27ee688a040..0a2dc0754c09 100644 --- a/mlir/python/CMakeLists.txt +++ b/mlir/python/CMakeLists.txt @@ -524,6 +524,19 @@ declare_mlir_python_extension(MLIRPythonExtension.Dialects.Quant.Pybind MLIRCAPIQuant ) +declare_mlir_python_extension(MLIRPythonExtension.Dialects.NVGPU.Pybind + MODULE_NAME _mlirDialectsNvgpu + ADD_TO_PARENT MLIRPythonSources.Dialects.nvgpu + ROOT_DIR "${PYTHON_SOURCE_DIR}" + SOURCES + DialectNVGPU.cpp + PRIVATE_LINK_LIBS + LLVMSupport + EMBED_CAPI_LINK_LIBS + MLIRCAPIIR + MLIRCAPINVGPU +) + declare_mlir_python_extension(MLIRPythonExtension.Dialects.PDL.Pybind MODULE_NAME _mlirDialectsPDL ADD_TO_PARENT MLIRPythonSources.Dialects.pdl diff --git a/mlir/python/mlir/dialects/nvgpu.py b/mlir/python/mlir/dialects/nvgpu.py index 2f6993b768ca..e19bf610ea33 100644 --- a/mlir/python/mlir/dialects/nvgpu.py +++ b/mlir/python/mlir/dialects/nvgpu.py @@ -4,3 +4,4 @@ from ._nvgpu_ops_gen import * from ._nvgpu_enum_gen import * +from .._mlir_libs._mlirDialectsNvgpu import * diff --git a/mlir/test/python/dialects/nvgpu.py b/mlir/test/python/dialects/nvgpu.py index 3158388f0e68..6df32bdd3c27 100644 --- a/mlir/test/python/dialects/nvgpu.py +++ b/mlir/test/python/dialects/nvgpu.py @@ -15,6 +15,23 @@ def constructAndPrintInModule(f): return f +# CHECK-LABEL: testTypes +@constructAndPrintInModule +def testTypes(): + tensorMemrefType = MemRefType.get( + (128, 64), F16Type.get(), memory_space=Attribute.parse("3") + ) + # CHECK: !nvgpu.tensormap.descriptor, swizzle = swizzle_128b, l2promo = l2promo_256b, oob = nan, interleave = none> + tma_desc = nvgpu.TensorMapDescriptorType.get( + tensorMemrefType, + nvgpu.TensorMapSwizzleKind.SWIZZLE_128B, + nvgpu.TensorMapL2PromoKind.L2PROMO_256B, + nvgpu.TensorMapOOBKind.OOB_NAN, + nvgpu.TensorMapInterleaveKind.INTERLEAVE_NONE, + ) + print(tma_desc) + + # CHECK-LABEL: testSmoke @constructAndPrintInModule def testSmoke(): -- GitLab From fda04b1caaf1a61b208f23e717a2db6d9b861f5a Mon Sep 17 00:00:00 2001 From: Rajveer Singh Bharadwaj Date: Wed, 17 Apr 2024 19:37:18 +0530 Subject: [PATCH 057/862] [libc] Replace mentions of `LIBC_FULLBUILD` with `LLVM_LIBC_FULL_BUILD` in 'examples/' (#88657) Resolves #88328 --- libc/examples/README.md | 2 +- libc/examples/examples.cmake | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libc/examples/README.md b/libc/examples/README.md index 36b886090c6c..1bc4a67294f2 100644 --- a/libc/examples/README.md +++ b/libc/examples/README.md @@ -59,7 +59,7 @@ have installed them, you have to inform CMake that we are linking against the full libc as follows: ```bash -cmake ../ -G -DLIBC_FULLBUILD=ON \ +cmake ../ -G -DLLVM_LIBC_FULL_BUILD=ON \ -DCMAKE_SYSROOT= \ -DCMAKE_C_COMPILER=/bin/clang \ -DCMAKE_TRY_COMPILE_TARGET_TYPE=STATIC_LIBRARY diff --git a/libc/examples/examples.cmake b/libc/examples/examples.cmake index 81e99e3cbede..6bb6b41c252f 100644 --- a/libc/examples/examples.cmake +++ b/libc/examples/examples.cmake @@ -4,13 +4,13 @@ function(add_example name) ${ARGN} ) - if(LIBC_FULLBUILD) + if(LLVM_LIBC_FULL_BUILD) target_link_options(${name} PRIVATE -static -rtlib=compiler-rt -fuse-ld=lld) elseif(LIBC_OVERLAY_ARCHIVE_DIR) target_link_directories(${name} PRIVATE ${LIBC_OVERLAY_ARCHIVE_DIR}) target_link_options(${name} PRIVATE -l:libllvmlibc.a) else() - message(FATAL_ERROR "Either LIBC_FULLBUILD should be on or " + message(FATAL_ERROR "Either LLVM_LIBC_FULL_BUILD should be on or " "LIBC_OVERLAY_ARCHIVE_DIR should be set.") endif() endfunction() -- GitLab From d558c090fc78beb6737098f058a084635b893567 Mon Sep 17 00:00:00 2001 From: yronglin Date: Wed, 17 Apr 2024 22:11:04 +0800 Subject: [PATCH 058/862] [NFC] Clean dead code in ParsedAttr.h (#89064) Signed-off-by: yronglin --- clang/include/clang/Sema/ParsedAttr.h | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/clang/include/clang/Sema/ParsedAttr.h b/clang/include/clang/Sema/ParsedAttr.h index e3857b2f07d9..25a5fa05b21c 100644 --- a/clang/include/clang/Sema/ParsedAttr.h +++ b/clang/include/clang/Sema/ParsedAttr.h @@ -94,7 +94,7 @@ struct PropertyData { : GetterId(getterId), SetterId(setterId) {} }; -} // namespace +} // namespace detail /// Wraps an identifier and optional source location for the identifier. struct IdentifierLoc { @@ -743,11 +743,6 @@ public: IdentifierInfo *scopeName, SourceLocation scopeLoc, ArgsUnion *args, unsigned numArgs, ParsedAttr::Form form, SourceLocation ellipsisLoc = SourceLocation()) { - size_t temp = - ParsedAttr::totalSizeToAlloc(numArgs, 0, 0, 0, 0); - (void)temp; void *memory = allocate( ParsedAttr::totalSizeToAlloc Date: Wed, 17 Apr 2024 14:29:09 +0000 Subject: [PATCH 059/862] [lldb] XFAIL TestDetachResumes on windows --- .../API/commands/process/detach-resumes/TestDetachResumes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lldb/test/API/commands/process/detach-resumes/TestDetachResumes.py b/lldb/test/API/commands/process/detach-resumes/TestDetachResumes.py index 57727294ddc3..797db55a45b9 100644 --- a/lldb/test/API/commands/process/detach-resumes/TestDetachResumes.py +++ b/lldb/test/API/commands/process/detach-resumes/TestDetachResumes.py @@ -12,6 +12,7 @@ from lldbsuite.test import lldbutil class DetachResumesTestCase(TestBase): NO_DEBUG_INFO_TESTCASE = True + @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr89077") def test_detach_resumes(self): self.build() exe = self.getBuildArtifact() -- GitLab From e49043512dbdc68319093da46e95a1e331ef837e Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Wed, 17 Apr 2024 14:55:53 +0100 Subject: [PATCH 060/862] [CostModel][X86] Update BITREVERSE costs for GFNI targets Inspired by the recent patches by @shamithoke - we have real scheduler model numbers for GFNI instructions now, allowing us to calculate an upper bounds costs table instead of performing it analytically. --- .../lib/Target/X86/X86TargetTransformInfo.cpp | 41 +++-- .../CostModel/X86/bitreverse-codesize.ll | 64 +++---- .../CostModel/X86/bitreverse-latency.ll | 160 +++++++++--------- .../CostModel/X86/bitreverse-sizelatency.ll | 160 +++++++++--------- .../test/Analysis/CostModel/X86/bitreverse.ll | 136 +++++++-------- 5 files changed, 284 insertions(+), 277 deletions(-) diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp index b466624e1334..38064f979269 100644 --- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp @@ -3820,6 +3820,24 @@ X86TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, { ISD::FSQRT, MVT::v2f64, { 27, 27, 1, 1 } }, // vsqrtpd { ISD::FSQRT, MVT::v4f64, { 54, 54, 1, 3 } }, // vsqrtpd }; + static const CostKindTblEntry GFNICostTbl[] = { + { ISD::BITREVERSE, MVT::i8, { 3, 3, 3, 4 } }, // gf2p8affineqb + { ISD::BITREVERSE, MVT::i16, { 3, 3, 4, 6 } }, // gf2p8affineqb + { ISD::BITREVERSE, MVT::i32, { 3, 3, 4, 5 } }, // gf2p8affineqb + { ISD::BITREVERSE, MVT::i64, { 3, 3, 4, 6 } }, // gf2p8affineqb + { ISD::BITREVERSE, MVT::v16i8, { 1, 6, 1, 2 } }, // gf2p8affineqb + { ISD::BITREVERSE, MVT::v32i8, { 1, 6, 1, 2 } }, // gf2p8affineqb + { ISD::BITREVERSE, MVT::v64i8, { 1, 6, 1, 2 } }, // gf2p8affineqb + { ISD::BITREVERSE, MVT::v8i16, { 1, 8, 2, 4 } }, // gf2p8affineqb + { ISD::BITREVERSE, MVT::v16i16, { 1, 9, 2, 4 } }, // gf2p8affineqb + { ISD::BITREVERSE, MVT::v32i16, { 1, 9, 2, 4 } }, // gf2p8affineqb + { ISD::BITREVERSE, MVT::v4i32, { 1, 8, 2, 4 } }, // gf2p8affineqb + { ISD::BITREVERSE, MVT::v8i32, { 1, 9, 2, 4 } }, // gf2p8affineqb + { ISD::BITREVERSE, MVT::v16i32, { 1, 9, 2, 4 } }, // gf2p8affineqb + { ISD::BITREVERSE, MVT::v2i64, { 1, 8, 2, 4 } }, // gf2p8affineqb + { ISD::BITREVERSE, MVT::v4i64, { 1, 9, 2, 4 } }, // gf2p8affineqb + { ISD::BITREVERSE, MVT::v8i64, { 1, 9, 2, 4 } }, // gf2p8affineqb + }; static const CostKindTblEntry GLMCostTbl[] = { { ISD::FSQRT, MVT::f32, { 19, 20, 1, 1 } }, // sqrtss { ISD::FSQRT, MVT::v4f32, { 37, 41, 1, 5 } }, // sqrtps @@ -4156,23 +4174,6 @@ X86TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, std::pair LT = getTypeLegalizationCost(OpTy); MVT MTy = LT.second; - // Attempt to lookup cost. - if (ISD == ISD::BITREVERSE && ST->hasGFNI() && ST->hasSSSE3() && - MTy.isVector()) { - // With PSHUFB the code is very similar for all types. If we have integer - // byte operations, we just need a GF2P8AFFINEQB for vXi8. For other types - // we also need a PSHUFB. - unsigned Cost = MTy.getVectorElementType() == MVT::i8 ? 1 : 2; - - // Without byte operations, we need twice as many GF2P8AFFINEQB and PSHUFB - // instructions. We also need an extract and an insert. - if (!(MTy.is128BitVector() || (ST->hasAVX2() && MTy.is256BitVector()) || - (ST->hasBWI() && MTy.is512BitVector()))) - Cost = Cost * 2 + 2; - - return LT.first * Cost; - } - // Without BMI/LZCNT see if we're only looking for a *_ZERO_UNDEF cost. if (((ISD == ISD::CTTZ && !ST->hasBMI()) || (ISD == ISD::CTLZ && !ST->hasLZCNT())) && @@ -4230,6 +4231,12 @@ X86TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, return adjustTableCost(Entry->ISD, *KindCost, LT.first, ICA.getFlags()); + if (ST->hasGFNI()) + if (const auto *Entry = CostTableLookup(GFNICostTbl, ISD, MTy)) + if (auto KindCost = Entry->Cost[CostKind]) + return adjustTableCost(Entry->ISD, *KindCost, LT.first, + ICA.getFlags()); + if (ST->hasCDI()) if (const auto *Entry = CostTableLookup(AVX512CDCostTbl, ISD, MTy)) if (auto KindCost = Entry->Cost[CostKind]) diff --git a/llvm/test/Analysis/CostModel/X86/bitreverse-codesize.ll b/llvm/test/Analysis/CostModel/X86/bitreverse-codesize.ll index 90e49e6ccb81..e02ba761aa8d 100644 --- a/llvm/test/Analysis/CostModel/X86/bitreverse-codesize.ll +++ b/llvm/test/Analysis/CostModel/X86/bitreverse-codesize.ll @@ -40,23 +40,23 @@ define i64 @var_bitreverse_i64(i64 %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_i64' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_i64' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_i64' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_i64' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_i64' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) @@ -77,23 +77,23 @@ define i32 @var_bitreverse_i32(i32 %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_i32' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_i32' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_i32' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_i32' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_i32' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) @@ -114,23 +114,23 @@ define i16 @var_bitreverse_i16(i16 %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_i16' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_i16' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_i16' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_i16' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_i16' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) @@ -151,23 +151,23 @@ define i8 @var_bitreverse_i8(i8 %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_i8' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_i8' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_i8' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_i8' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_i8' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) @@ -270,7 +270,7 @@ define <4 x i64> @var_bitreverse_v4i64(<4 x i64> %a) { ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v4i64' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v4i64' @@ -323,7 +323,7 @@ define <8 x i64> @var_bitreverse_v8i64(<8 x i64> %a) { ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v8i64' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v8i64' @@ -331,7 +331,7 @@ define <8 x i64> @var_bitreverse_v8i64(<8 x i64> %a) { ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v8i64' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v8i64' @@ -421,7 +421,7 @@ define <8 x i32> @var_bitreverse_v8i32(<8 x i32> %a) { ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v8i32' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v8i32' @@ -474,7 +474,7 @@ define <16 x i32> @var_bitreverse_v16i32(<16 x i32> %a) { ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v16i32' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v16i32' @@ -482,7 +482,7 @@ define <16 x i32> @var_bitreverse_v16i32(<16 x i32> %a) { ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v16i32' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v16i32' @@ -572,7 +572,7 @@ define <16 x i16> @var_bitreverse_v16i16(<16 x i16> %a) { ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v16i16' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v16i16' @@ -625,7 +625,7 @@ define <32 x i16> @var_bitreverse_v32i16(<32 x i16> %a) { ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v32i16' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v32i16' @@ -633,7 +633,7 @@ define <32 x i16> @var_bitreverse_v32i16(<32 x i16> %a) { ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v32i16' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v32i16' @@ -723,7 +723,7 @@ define <32 x i8> @var_bitreverse_v32i8(<32 x i8> %a) { ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v32i8' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v32i8' @@ -776,7 +776,7 @@ define <64 x i8> @var_bitreverse_v64i8(<64 x i8> %a) { ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v64i8' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v64i8' @@ -784,7 +784,7 @@ define <64 x i8> @var_bitreverse_v64i8(<64 x i8> %a) { ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v64i8' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v64i8' diff --git a/llvm/test/Analysis/CostModel/X86/bitreverse-latency.ll b/llvm/test/Analysis/CostModel/X86/bitreverse-latency.ll index e7d0d05f8242..ba231b985c26 100644 --- a/llvm/test/Analysis/CostModel/X86/bitreverse-latency.ll +++ b/llvm/test/Analysis/CostModel/X86/bitreverse-latency.ll @@ -40,23 +40,23 @@ define i64 @var_bitreverse_i64(i64 %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_i64' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_i64' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_i64' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_i64' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_i64' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) @@ -77,23 +77,23 @@ define i32 @var_bitreverse_i32(i32 %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_i32' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_i32' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_i32' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_i32' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_i32' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) @@ -114,23 +114,23 @@ define i16 @var_bitreverse_i16(i16 %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_i16' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_i16' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_i16' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_i16' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_i16' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) @@ -151,23 +151,23 @@ define i8 @var_bitreverse_i8(i8 %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_i8' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_i8' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_i8' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_i8' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_i8' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) @@ -221,23 +221,23 @@ define <2 x i64> @var_bitreverse_v2i64(<2 x i64> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v2i64' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v2i64' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v2i64' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v2i64' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v2i64' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %bitreverse ; %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) @@ -270,23 +270,23 @@ define <4 x i64> @var_bitreverse_v4i64(<4 x i64> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v4i64' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v4i64' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v4i64' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v4i64' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v4i64' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %bitreverse ; %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) @@ -323,23 +323,23 @@ define <8 x i64> @var_bitreverse_v8i64(<8 x i64> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v8i64' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v8i64' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v8i64' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v8i64' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v8i64' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %bitreverse ; %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) @@ -376,23 +376,23 @@ define <4 x i32> @var_bitreverse_v4i32(<4 x i32> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v4i32' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v4i32' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v4i32' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v4i32' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v4i32' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %bitreverse ; %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) @@ -425,23 +425,23 @@ define <8 x i32> @var_bitreverse_v8i32(<8 x i32> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v8i32' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v8i32' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v8i32' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v8i32' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v8i32' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %bitreverse ; %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) @@ -478,23 +478,23 @@ define <16 x i32> @var_bitreverse_v16i32(<16 x i32> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v16i32' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v16i32' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v16i32' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v16i32' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v16i32' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %bitreverse ; %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) @@ -531,23 +531,23 @@ define <8 x i16> @var_bitreverse_v8i16(<8 x i16> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v8i16' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v8i16' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v8i16' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v8i16' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v8i16' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %bitreverse ; %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) @@ -580,23 +580,23 @@ define <16 x i16> @var_bitreverse_v16i16(<16 x i16> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v16i16' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v16i16' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v16i16' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v16i16' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v16i16' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %bitreverse ; %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) @@ -633,23 +633,23 @@ define <32 x i16> @var_bitreverse_v32i16(<32 x i16> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v32i16' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v32i16' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v32i16' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v32i16' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v32i16' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %bitreverse ; %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) @@ -686,23 +686,23 @@ define <16 x i8> @var_bitreverse_v16i8(<16 x i8> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v16i8' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v16i8' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v16i8' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v16i8' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v16i8' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %bitreverse ; %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) @@ -735,23 +735,23 @@ define <32 x i8> @var_bitreverse_v32i8(<32 x i8> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v32i8' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v32i8' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v32i8' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v32i8' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v32i8' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %bitreverse ; %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) @@ -788,23 +788,23 @@ define <64 x i8> @var_bitreverse_v64i8(<64 x i8> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v64i8' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v64i8' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v64i8' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v64i8' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v64i8' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %bitreverse ; %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) diff --git a/llvm/test/Analysis/CostModel/X86/bitreverse-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/bitreverse-sizelatency.ll index 1ba93af46ebd..d60fac228fc0 100644 --- a/llvm/test/Analysis/CostModel/X86/bitreverse-sizelatency.ll +++ b/llvm/test/Analysis/CostModel/X86/bitreverse-sizelatency.ll @@ -40,23 +40,23 @@ define i64 @var_bitreverse_i64(i64 %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_i64' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_i64' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_i64' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_i64' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_i64' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) @@ -77,23 +77,23 @@ define i32 @var_bitreverse_i32(i32 %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_i32' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_i32' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_i32' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_i32' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_i32' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) @@ -114,23 +114,23 @@ define i16 @var_bitreverse_i16(i16 %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_i16' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_i16' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_i16' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_i16' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_i16' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) @@ -151,23 +151,23 @@ define i8 @var_bitreverse_i8(i8 %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_i8' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_i8' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_i8' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_i8' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_i8' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) @@ -217,23 +217,23 @@ define <2 x i64> @var_bitreverse_v2i64(<2 x i64> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v2i64' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v2i64' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v2i64' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v2i64' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v2i64' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %bitreverse ; %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) @@ -270,23 +270,23 @@ define <4 x i64> @var_bitreverse_v4i64(<4 x i64> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v4i64' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v4i64' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v4i64' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v4i64' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v4i64' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %bitreverse ; %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) @@ -323,23 +323,23 @@ define <8 x i64> @var_bitreverse_v8i64(<8 x i64> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v8i64' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v8i64' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v8i64' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v8i64' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v8i64' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %bitreverse ; %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) @@ -372,23 +372,23 @@ define <4 x i32> @var_bitreverse_v4i32(<4 x i32> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v4i32' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v4i32' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v4i32' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v4i32' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v4i32' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %bitreverse ; %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) @@ -425,23 +425,23 @@ define <8 x i32> @var_bitreverse_v8i32(<8 x i32> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v8i32' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v8i32' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v8i32' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v8i32' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v8i32' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %bitreverse ; %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) @@ -478,23 +478,23 @@ define <16 x i32> @var_bitreverse_v16i32(<16 x i32> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v16i32' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v16i32' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v16i32' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v16i32' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v16i32' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %bitreverse ; %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) @@ -527,23 +527,23 @@ define <8 x i16> @var_bitreverse_v8i16(<8 x i16> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v8i16' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v8i16' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v8i16' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v8i16' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v8i16' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %bitreverse ; %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) @@ -580,23 +580,23 @@ define <16 x i16> @var_bitreverse_v16i16(<16 x i16> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v16i16' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v16i16' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v16i16' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v16i16' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v16i16' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %bitreverse ; %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) @@ -633,23 +633,23 @@ define <32 x i16> @var_bitreverse_v32i16(<32 x i16> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v32i16' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v32i16' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v32i16' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v32i16' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v32i16' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %bitreverse ; %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) @@ -682,23 +682,23 @@ define <16 x i8> @var_bitreverse_v16i8(<16 x i8> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v16i8' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v16i8' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v16i8' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v16i8' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v16i8' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %bitreverse ; %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) @@ -735,23 +735,23 @@ define <32 x i8> @var_bitreverse_v32i8(<32 x i8> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v32i8' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v32i8' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v32i8' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v32i8' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v32i8' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %bitreverse ; %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) @@ -788,23 +788,23 @@ define <64 x i8> @var_bitreverse_v64i8(<64 x i8> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v64i8' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v64i8' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v64i8' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v64i8' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v64i8' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %bitreverse ; %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) diff --git a/llvm/test/Analysis/CostModel/X86/bitreverse.ll b/llvm/test/Analysis/CostModel/X86/bitreverse.ll index 0b76b0d527ba..a890147fee46 100644 --- a/llvm/test/Analysis/CostModel/X86/bitreverse.ll +++ b/llvm/test/Analysis/CostModel/X86/bitreverse.ll @@ -40,23 +40,23 @@ define i64 @var_bitreverse_i64(i64 %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i64 %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_i64' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i64 %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_i64' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i64 %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_i64' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i64 %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_i64' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i64 %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_i64' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i64 %bitreverse ; %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) @@ -77,23 +77,23 @@ define i32 @var_bitreverse_i32(i32 %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_i32' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_i32' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_i32' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_i32' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_i32' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 %bitreverse ; %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) @@ -114,23 +114,23 @@ define i16 @var_bitreverse_i16(i16 %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i16 %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_i16' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i16 %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_i16' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i16 %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_i16' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i16 %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_i16' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i16 %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_i16' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i16 %bitreverse ; %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) @@ -151,23 +151,23 @@ define i8 @var_bitreverse_i8(i8 %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i8 %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_i8' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i8 %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_i8' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i8 %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_i8' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i8 %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_i8' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i8 %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_i8' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i8 %bitreverse ; %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) @@ -217,23 +217,23 @@ define <2 x i64> @var_bitreverse_v2i64(<2 x i64> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v2i64' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v2i64' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v2i64' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v2i64' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v2i64' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %bitreverse ; %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) @@ -270,23 +270,23 @@ define <4 x i64> @var_bitreverse_v4i64(<4 x i64> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i64> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v4i64' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i64> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v4i64' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i64> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v4i64' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i64> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v4i64' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i64> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v4i64' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i64> %bitreverse ; %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) @@ -323,23 +323,23 @@ define <8 x i64> @var_bitreverse_v8i64(<8 x i64> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v8i64' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v8i64' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v8i64' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v8i64' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v8i64' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %bitreverse ; %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) @@ -372,23 +372,23 @@ define <4 x i32> @var_bitreverse_v4i32(<4 x i32> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v4i32' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v4i32' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v4i32' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v4i32' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v4i32' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %bitreverse ; %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) @@ -425,23 +425,23 @@ define <8 x i32> @var_bitreverse_v8i32(<8 x i32> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i32> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v8i32' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i32> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v8i32' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i32> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v8i32' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i32> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v8i32' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i32> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v8i32' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i32> %bitreverse ; %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) @@ -478,23 +478,23 @@ define <16 x i32> @var_bitreverse_v16i32(<16 x i32> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i32> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v16i32' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i32> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v16i32' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i32> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v16i32' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i32> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v16i32' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i32> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v16i32' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i32> %bitreverse ; %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) @@ -527,23 +527,23 @@ define <8 x i16> @var_bitreverse_v8i16(<8 x i16> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v8i16' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v8i16' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v8i16' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v8i16' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v8i16' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %bitreverse ; %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) @@ -580,23 +580,23 @@ define <16 x i16> @var_bitreverse_v16i16(<16 x i16> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v16i16' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v16i16' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v16i16' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v16i16' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v16i16' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %bitreverse ; %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) @@ -633,23 +633,23 @@ define <32 x i16> @var_bitreverse_v32i16(<32 x i16> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v32i16' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v32i16' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v32i16' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v32i16' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v32i16' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %bitreverse ; %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) @@ -743,7 +743,7 @@ define <32 x i8> @var_bitreverse_v32i8(<32 x i8> %a) { ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v32i8' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v32i8' @@ -796,7 +796,7 @@ define <64 x i8> @var_bitreverse_v64i8(<64 x i8> %a) { ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v64i8' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v64i8' @@ -804,7 +804,7 @@ define <64 x i8> @var_bitreverse_v64i8(<64 x i8> %a) { ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v64i8' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v64i8' -- GitLab From 4a5ab13bf5a94ec7f0eabaf24dfe1a5ee720b860 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Wed, 17 Apr 2024 15:46:24 +0100 Subject: [PATCH 061/862] [VectorCombine] Remove single quotes from "-passes=vector-combine" These confuse the update_test_checks.py script when run by DOS cmd.exe --- llvm/test/Transforms/VectorCombine/AArch64/select-shuffle.ll | 2 +- llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll | 2 +- llvm/test/Transforms/VectorCombine/AArch64/vecreduce-shuffle.ll | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/llvm/test/Transforms/VectorCombine/AArch64/select-shuffle.ll b/llvm/test/Transforms/VectorCombine/AArch64/select-shuffle.ll index eae087900483..b27c026dbccf 100644 --- a/llvm/test/Transforms/VectorCombine/AArch64/select-shuffle.ll +++ b/llvm/test/Transforms/VectorCombine/AArch64/select-shuffle.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -passes='vector-combine' -S %s | FileCheck %s +; RUN: opt -passes=vector-combine -S %s | FileCheck %s target triple = "aarch64" diff --git a/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll b/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll index d96dfec84916..a1d4ca1a740e 100644 --- a/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll +++ b/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -passes='vector-combine' -S %s | FileCheck %s +; RUN: opt -passes=vector-combine -S %s | FileCheck %s target triple = "aarch64" diff --git a/llvm/test/Transforms/VectorCombine/AArch64/vecreduce-shuffle.ll b/llvm/test/Transforms/VectorCombine/AArch64/vecreduce-shuffle.ll index d69cb75664a8..a22575ccb1ca 100644 --- a/llvm/test/Transforms/VectorCombine/AArch64/vecreduce-shuffle.ll +++ b/llvm/test/Transforms/VectorCombine/AArch64/vecreduce-shuffle.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -passes='vector-combine' -S %s | FileCheck %s +; RUN: opt -passes=vector-combine -S %s | FileCheck %s target triple = "aarch64" -- GitLab From 5d314353fbec1a15cd8900f466dcdcf2af40e8c9 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Wed, 17 Apr 2024 15:53:15 +0100 Subject: [PATCH 062/862] [VPlan] Check for VPWidenLoadRecipe directly in truncateToMinBW. (NFCI). Since ne After a separate recipe has been introduced for wide loads in a9bafe91dd0, we can directly check for load recipes in the early bail-out and remove the redundant bail out for stores. --- llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 78d0b5b95c5e..901ecd10c69d 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -975,9 +975,7 @@ void VPlanTransforms::truncateToMinimalBitwidths( vp_depth_first_deep(Plan.getVectorLoopRegion()))) { for (VPRecipeBase &R : make_early_inc_range(*VPBB)) { if (!isa(&R)) - continue; - if (isa(&R)) + VPWidenSelectRecipe, VPWidenLoadRecipe>(&R)) continue; VPValue *ResultVPV = R.getVPSingleValue(); -- GitLab From 812963f6aa2adb5e990f273b8ce1a0eabcdefd7f Mon Sep 17 00:00:00 2001 From: Mark de Wever Date: Wed, 17 Apr 2024 17:02:07 +0200 Subject: [PATCH 063/862] [libc++][chrono] Improves date formatting. (#86127) The formatting of years has been done manually since the results of %Y outside the "typical" range may produce unexpected values. The same applies to %F which is identical to %Y-%m-%d. None of these conversion specifiers is affected by the locale used. So it's trivial to manually handle this case. This removes several platform specific ifdefs from the tests. --- libcxx/include/__chrono/formatter.h | 16 ++++----- .../time.cal.ymd.nonmembers/ostream.pass.cpp | 34 ------------------- .../sys_date.ostream.pass.cpp | 34 ------------------- .../formatter.year_month_day.pass.cpp | 24 ------------- 4 files changed, 7 insertions(+), 101 deletions(-) diff --git a/libcxx/include/__chrono/formatter.h b/libcxx/include/__chrono/formatter.h index d932a99f4b99..f76e7b2ea0e8 100644 --- a/libcxx/include/__chrono/formatter.h +++ b/libcxx/include/__chrono/formatter.h @@ -322,15 +322,13 @@ _LIBCPP_HIDE_FROM_ABI void __format_chrono_using_chrono_specs( __formatter::__format_year(__sstr, __t.tm_year + 1900); break; - case _CharT('F'): { - int __year = __t.tm_year + 1900; - if (__year < 1000) { - __formatter::__format_year(__sstr, __year); - __sstr << std::format(_LIBCPP_STATICALLY_WIDEN(_CharT, "-{:02}-{:02}"), __t.tm_mon + 1, __t.tm_mday); - } else - __facet.put( - {__sstr}, __sstr, _CharT(' '), std::addressof(__t), std::to_address(__s), std::to_address(__it + 1)); - } break; + case _CharT('F'): + // Depending on the platform's libc the range of supported years is + // limited. Instead of testing all conditions use the internal + // implementation unconditionally. + __formatter::__format_year(__sstr, __t.tm_year + 1900); + __sstr << std::format(_LIBCPP_STATICALLY_WIDEN(_CharT, "-{:02}-{:02}"), __t.tm_mon + 1, __t.tm_mday); + break; case _CharT('z'): __formatter::__format_zone_offset(__sstr, __z.__offset, false); diff --git a/libcxx/test/std/time/time.cal/time.cal.ymd/time.cal.ymd.nonmembers/ostream.pass.cpp b/libcxx/test/std/time/time.cal/time.cal.ymd/time.cal.ymd.nonmembers/ostream.pass.cpp index ffc737fcad5d..20ffb165558e 100644 --- a/libcxx/test/std/time/time.cal/time.cal.ymd/time.cal.ymd.nonmembers/ostream.pass.cpp +++ b/libcxx/test/std/time/time.cal/time.cal.ymd/time.cal.ymd.nonmembers/ostream.pass.cpp @@ -13,9 +13,6 @@ // TODO FMT This test should not require std::to_chars(floating-point) // XFAIL: availability-fp_to_chars-missing -// TODO FMT Investigate Windows issues. -// XFAIL: msvc - // REQUIRES: locale.fr_FR.UTF-8 // REQUIRES: locale.ja_JP.UTF-8 @@ -89,20 +86,9 @@ static void test() { TEST_EQUAL(stream_c_locale( std::chrono::year_month_day{std::chrono::year{2000}, std::chrono::month{2}, std::chrono::day{29}}), SV("2000-02-29")); - -#if defined(_AIX) - TEST_EQUAL(stream_c_locale( - std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}), - SV("+32767-12-31")); -#elif defined(_WIN32) // defined(_AIX) - TEST_EQUAL(stream_c_locale( - std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}), - SV("")); -#else // defined(_AIX) TEST_EQUAL(stream_c_locale( std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}), SV("32767-12-31")); -#endif // defined(_AIX) TEST_EQUAL(stream_fr_FR_locale( std::chrono::year_month_day{std::chrono::year{-32'768}, std::chrono::month{1}, std::chrono::day{1}}), @@ -122,19 +108,9 @@ static void test() { TEST_EQUAL(stream_fr_FR_locale( std::chrono::year_month_day{std::chrono::year{2000}, std::chrono::month{2}, std::chrono::day{29}}), SV("2000-02-29")); -#if defined(_AIX) - TEST_EQUAL(stream_fr_FR_locale( - std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}), - SV("+32767-12-31")); -#elif defined(_WIN32) // defined(_AIX) - TEST_EQUAL(stream_fr_FR_locale( - std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}), - SV("")); -#else // defined(_AIX) TEST_EQUAL(stream_fr_FR_locale( std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}), SV("32767-12-31")); -#endif // defined(_AIX) TEST_EQUAL(stream_ja_JP_locale( std::chrono::year_month_day{std::chrono::year{-32'768}, std::chrono::month{1}, std::chrono::day{1}}), @@ -154,19 +130,9 @@ static void test() { TEST_EQUAL(stream_ja_JP_locale( std::chrono::year_month_day{std::chrono::year{2000}, std::chrono::month{2}, std::chrono::day{29}}), SV("2000-02-29")); -#if defined(_AIX) - TEST_EQUAL(stream_ja_JP_locale( - std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}), - SV("+32767-12-31")); -#elif defined(_WIN32) // defined(_AIX) - TEST_EQUAL(stream_ja_JP_locale( - std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}), - SV("")); -#else // defined(_AIX) TEST_EQUAL(stream_ja_JP_locale( std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}), SV("32767-12-31")); -#endif // defined(_AIX) } int main(int, char**) { diff --git a/libcxx/test/std/time/time.clock/time.clock.system/sys_date.ostream.pass.cpp b/libcxx/test/std/time/time.clock/time.clock.system/sys_date.ostream.pass.cpp index 7af3ebf77680..c5645884cfed 100644 --- a/libcxx/test/std/time/time.clock/time.clock.system/sys_date.ostream.pass.cpp +++ b/libcxx/test/std/time/time.clock/time.clock.system/sys_date.ostream.pass.cpp @@ -13,9 +13,6 @@ // TODO FMT This test should not require std::to_chars(floating-point) // XFAIL: availability-fp_to_chars-missing -// TODO FMT Investigate Windows issues. -// XFAIL: msvc - // REQUIRES: locale.fr_FR.UTF-8 // REQUIRES: locale.ja_JP.UTF-8 @@ -81,20 +78,9 @@ static void test() { TEST_EQUAL(stream_c_locale(std::chrono::sys_days{ std::chrono::year_month_day{std::chrono::year{2000}, std::chrono::month{2}, std::chrono::day{29}}}), SV("2000-02-29")); - -#if defined(_AIX) - TEST_EQUAL(stream_c_locale(std::chrono::sys_days{ - std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}}), - SV("+32767-12-31")); -#elif defined(_WIN32) // defined(_AIX) - TEST_EQUAL(stream_c_locale(std::chrono::sys_days{ - std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}}), - SV("")); -#else // defined(_AIX) TEST_EQUAL(stream_c_locale(std::chrono::sys_days{ std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}}), SV("32767-12-31")); -#endif // defined(_AIX) // multiples of days are considered days. TEST_EQUAL(stream_c_locale(std::chrono::sys_time{std::chrono::weeks{3}}), @@ -112,19 +98,9 @@ static void test() { TEST_EQUAL(stream_fr_FR_locale(std::chrono::sys_days{ std::chrono::year_month_day{std::chrono::year{2000}, std::chrono::month{2}, std::chrono::day{29}}}), SV("2000-02-29")); -#if defined(_AIX) - TEST_EQUAL(stream_fr_FR_locale(std::chrono::sys_days{ - std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}}), - SV("+32767-12-31")); -#elif defined(_WIN32) // defined(_AIX) - TEST_EQUAL(stream_fr_FR_locale(std::chrono::sys_days{ - std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}}), - SV("")); -#else // defined(_AIX) TEST_EQUAL(stream_fr_FR_locale(std::chrono::sys_days{ std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}}), SV("32767-12-31")); -#endif // defined(_AIX) // multiples of days are considered days. TEST_EQUAL(stream_fr_FR_locale(std::chrono::sys_time{std::chrono::weeks{3}}), @@ -142,19 +118,9 @@ static void test() { TEST_EQUAL(stream_ja_JP_locale(std::chrono::sys_days{ std::chrono::year_month_day{std::chrono::year{2000}, std::chrono::month{2}, std::chrono::day{29}}}), SV("2000-02-29")); -#if defined(_AIX) - TEST_EQUAL(stream_ja_JP_locale(std::chrono::sys_days{ - std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}}), - SV("+32767-12-31")); -#elif defined(_WIN32) // defined(_AIX) - TEST_EQUAL(stream_ja_JP_locale(std::chrono::sys_days{ - std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}}), - SV("")); -#else // defined(_AIX) TEST_EQUAL(stream_ja_JP_locale(std::chrono::sys_days{ std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}}), SV("32767-12-31")); -#endif // defined(_AIX) // multiples of days are considered days. TEST_EQUAL(stream_ja_JP_locale(std::chrono::sys_time{std::chrono::weeks{3}}), diff --git a/libcxx/test/std/time/time.syn/formatter.year_month_day.pass.cpp b/libcxx/test/std/time/time.syn/formatter.year_month_day.pass.cpp index 5a2b7afa37a8..1f2af1cb0530 100644 --- a/libcxx/test/std/time/time.syn/formatter.year_month_day.pass.cpp +++ b/libcxx/test/std/time/time.syn/formatter.year_month_day.pass.cpp @@ -62,17 +62,6 @@ static void test_no_chrono_specs() { std::chrono::year_month_day{std::chrono::year{1970}, std::chrono::month{2}, std::chrono::day{31}}); // Valid year, invalid month, valid day -#ifdef _WIN32 - check(SV(" is not a valid date"), - SV("{}"), - std::chrono::year_month_day{std::chrono::year{1970}, std::chrono::month{0}, std::chrono::day{31}}); - check(SV("****** is not a valid date******"), - SV("{:*^32}"), - std::chrono::year_month_day{std::chrono::year{1970}, std::chrono::month{0}, std::chrono::day{31}}); - check(SV("*********** is not a valid date"), - SV("{:*>31}"), - std::chrono::year_month_day{std::chrono::year{1970}, std::chrono::month{0}, std::chrono::day{31}}); -#else // _WIN32 check(SV("1970-00-31 is not a valid date"), SV("{}"), std::chrono::year_month_day{std::chrono::year{1970}, std::chrono::month{0}, std::chrono::day{31}}); @@ -82,20 +71,8 @@ static void test_no_chrono_specs() { check(SV("*1970-00-31 is not a valid date"), SV("{:*>31}"), std::chrono::year_month_day{std::chrono::year{1970}, std::chrono::month{0}, std::chrono::day{31}}); -#endif // _WIN32 // Valid year, invalid month, invalid day -#ifdef _WIN32 - check(SV(" is not a valid date"), - SV("{}"), - std::chrono::year_month_day{std::chrono::year{1970}, std::chrono::month{0}, std::chrono::day{32}}); - check(SV("****** is not a valid date******"), - SV("{:*^32}"), - std::chrono::year_month_day{std::chrono::year{1970}, std::chrono::month{0}, std::chrono::day{32}}); - check(SV("*********** is not a valid date"), - SV("{:*>31}"), - std::chrono::year_month_day{std::chrono::year{1970}, std::chrono::month{0}, std::chrono::day{32}}); -#else // _WIN32 check(SV("1970-00-32 is not a valid date"), SV("{}"), std::chrono::year_month_day{std::chrono::year{1970}, std::chrono::month{0}, std::chrono::day{32}}); @@ -105,7 +82,6 @@ static void test_no_chrono_specs() { check(SV("*1970-00-32 is not a valid date"), SV("{:*>31}"), std::chrono::year_month_day{std::chrono::year{1970}, std::chrono::month{0}, std::chrono::day{32}}); -#endif // _WIN32 // Invalid year, valid month, valid day check(SV("-32768-01-31 is not a valid date"), -- GitLab From 458328ae23d318a5055d5bac66426b8551bce01f Mon Sep 17 00:00:00 2001 From: Vlad Serebrennikov Date: Wed, 17 Apr 2024 18:01:55 +0300 Subject: [PATCH 064/862] [clang][NFC] Refactor `Sema::RedeclarationKind` This patch converts the enum into scoped enum, and moves it into its own header for the time being. It's definition is needed in `Sema.h`, and is going to be needed in upcoming `SemaObjC.h`. `Lookup.h` can't hold it, because it includes `Sema.h`. --- clang/include/clang/Sema/Lookup.h | 43 ++++++++++--------- clang/include/clang/Sema/Redeclaration.h | 31 +++++++++++++ clang/include/clang/Sema/Sema.h | 40 ++++------------- clang/lib/Interpreter/Interpreter.cpp | 3 +- clang/lib/Interpreter/InterpreterUtils.cpp | 2 +- clang/lib/Sema/SemaDecl.cpp | 24 ++++++----- clang/lib/Sema/SemaDeclCXX.cpp | 28 ++++++------ clang/lib/Sema/SemaExprCXX.cpp | 2 +- clang/lib/Sema/SemaExprMember.cpp | 2 +- clang/lib/Sema/SemaLookup.cpp | 16 ++++++- clang/lib/Sema/SemaOpenMP.cpp | 2 +- clang/lib/Sema/SemaTemplate.cpp | 5 ++- .../lib/Sema/SemaTemplateInstantiateDecl.cpp | 8 ++-- 13 files changed, 118 insertions(+), 88 deletions(-) create mode 100644 clang/include/clang/Sema/Redeclaration.h diff --git a/clang/include/clang/Sema/Lookup.h b/clang/include/clang/Sema/Lookup.h index 2f2f2607a937..0db5b847038f 100644 --- a/clang/include/clang/Sema/Lookup.h +++ b/clang/include/clang/Sema/Lookup.h @@ -153,28 +153,30 @@ public: using iterator = UnresolvedSetImpl::iterator; - LookupResult(Sema &SemaRef, const DeclarationNameInfo &NameInfo, - Sema::LookupNameKind LookupKind, - Sema::RedeclarationKind Redecl = Sema::NotForRedeclaration) + LookupResult( + Sema &SemaRef, const DeclarationNameInfo &NameInfo, + Sema::LookupNameKind LookupKind, + RedeclarationKind Redecl = RedeclarationKind::NotForRedeclaration) : SemaPtr(&SemaRef), NameInfo(NameInfo), LookupKind(LookupKind), - Redecl(Redecl != Sema::NotForRedeclaration), - ExternalRedecl(Redecl == Sema::ForExternalRedeclaration), - DiagnoseAccess(Redecl == Sema::NotForRedeclaration), - DiagnoseAmbiguous(Redecl == Sema::NotForRedeclaration) { + Redecl(Redecl != RedeclarationKind::NotForRedeclaration), + ExternalRedecl(Redecl == RedeclarationKind::ForExternalRedeclaration), + DiagnoseAccess(Redecl == RedeclarationKind::NotForRedeclaration), + DiagnoseAmbiguous(Redecl == RedeclarationKind::NotForRedeclaration) { configure(); } // TODO: consider whether this constructor should be restricted to take // as input a const IdentifierInfo* (instead of Name), // forcing other cases towards the constructor taking a DNInfo. - LookupResult(Sema &SemaRef, DeclarationName Name, SourceLocation NameLoc, - Sema::LookupNameKind LookupKind, - Sema::RedeclarationKind Redecl = Sema::NotForRedeclaration) + LookupResult( + Sema &SemaRef, DeclarationName Name, SourceLocation NameLoc, + Sema::LookupNameKind LookupKind, + RedeclarationKind Redecl = RedeclarationKind::NotForRedeclaration) : SemaPtr(&SemaRef), NameInfo(Name, NameLoc), LookupKind(LookupKind), - Redecl(Redecl != Sema::NotForRedeclaration), - ExternalRedecl(Redecl == Sema::ForExternalRedeclaration), - DiagnoseAccess(Redecl == Sema::NotForRedeclaration), - DiagnoseAmbiguous(Redecl == Sema::NotForRedeclaration) { + Redecl(Redecl != RedeclarationKind::NotForRedeclaration), + ExternalRedecl(Redecl == RedeclarationKind::ForExternalRedeclaration), + DiagnoseAccess(Redecl == RedeclarationKind::NotForRedeclaration), + DiagnoseAmbiguous(Redecl == RedeclarationKind::NotForRedeclaration) { configure(); } @@ -285,9 +287,10 @@ public: return ExternalRedecl; } - Sema::RedeclarationKind redeclarationKind() const { - return ExternalRedecl ? Sema::ForExternalRedeclaration : - Redecl ? Sema::ForVisibleRedeclaration : Sema::NotForRedeclaration; + RedeclarationKind redeclarationKind() const { + return ExternalRedecl ? RedeclarationKind::ForExternalRedeclaration + : Redecl ? RedeclarationKind::ForVisibleRedeclaration + : RedeclarationKind::NotForRedeclaration; } /// Specify whether hidden declarations are visible, e.g., @@ -615,9 +618,9 @@ public: } /// Change this lookup's redeclaration kind. - void setRedeclarationKind(Sema::RedeclarationKind RK) { - Redecl = (RK != Sema::NotForRedeclaration); - ExternalRedecl = (RK == Sema::ForExternalRedeclaration); + void setRedeclarationKind(RedeclarationKind RK) { + Redecl = (RK != RedeclarationKind::NotForRedeclaration); + ExternalRedecl = (RK == RedeclarationKind::ForExternalRedeclaration); configure(); } diff --git a/clang/include/clang/Sema/Redeclaration.h b/clang/include/clang/Sema/Redeclaration.h new file mode 100644 index 000000000000..ae18b922f5cd --- /dev/null +++ b/clang/include/clang/Sema/Redeclaration.h @@ -0,0 +1,31 @@ +//===- Redeclaration.h - Redeclarations--------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines RedeclarationKind enum. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_SEMA_REDECLARATION_H +#define LLVM_CLANG_SEMA_REDECLARATION_H + +/// Specifies whether (or how) name lookup is being performed for a +/// redeclaration (vs. a reference). +enum class RedeclarationKind { + /// The lookup is a reference to this name that is not for the + /// purpose of redeclaring the name. + NotForRedeclaration = 0, + /// The lookup results will be used for redeclaration of a name, + /// if an entity by that name already exists and is visible. + ForVisibleRedeclaration, + /// The lookup results will be used for redeclaration of a name + /// with external linkage; non-visible lookup results with external linkage + /// may also be found. + ForExternalRedeclaration +}; + +#endif // LLVM_CLANG_SEMA_REDECLARATION_H \ No newline at end of file diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h index 281e3b91de1d..1e89dfc58d92 100644 --- a/clang/include/clang/Sema/Sema.h +++ b/clang/include/clang/Sema/Sema.h @@ -52,6 +52,7 @@ #include "clang/Sema/IdentifierResolver.h" #include "clang/Sema/ObjCMethodList.h" #include "clang/Sema/Ownership.h" +#include "clang/Sema/Redeclaration.h" #include "clang/Sema/Scope.h" #include "clang/Sema/SemaBase.h" #include "clang/Sema/SemaConcept.h" @@ -7443,40 +7444,17 @@ public: typedef std::function TypoRecoveryCallback; - /// Specifies whether (or how) name lookup is being performed for a - /// redeclaration (vs. a reference). - enum RedeclarationKind { - /// The lookup is a reference to this name that is not for the - /// purpose of redeclaring the name. - NotForRedeclaration = 0, - /// The lookup results will be used for redeclaration of a name, - /// if an entity by that name already exists and is visible. - ForVisibleRedeclaration, - /// The lookup results will be used for redeclaration of a name - /// with external linkage; non-visible lookup results with external linkage - /// may also be found. - ForExternalRedeclaration - }; - - RedeclarationKind forRedeclarationInCurContext() const { - // A declaration with an owning module for linkage can never link against - // anything that is not visible. We don't need to check linkage here; if - // the context has internal linkage, redeclaration lookup won't find things - // from other TUs, and we can't safely compute linkage yet in general. - if (cast(CurContext) - ->getOwningModuleForLinkage(/*IgnoreLinkage*/ true)) - return ForVisibleRedeclaration; - return ForExternalRedeclaration; - } + RedeclarationKind forRedeclarationInCurContext() const; /// Look up a name, looking for a single declaration. Return /// null if the results were absent, ambiguous, or overloaded. /// /// It is preferable to use the elaborated form and explicitly handle /// ambiguity and overloaded. - NamedDecl *LookupSingleName(Scope *S, DeclarationName Name, - SourceLocation Loc, LookupNameKind NameKind, - RedeclarationKind Redecl = NotForRedeclaration); + NamedDecl *LookupSingleName( + Scope *S, DeclarationName Name, SourceLocation Loc, + LookupNameKind NameKind, + RedeclarationKind Redecl = RedeclarationKind::NotForRedeclaration); bool LookupBuiltin(LookupResult &R); void LookupNecessaryTypesForBuiltin(Scope *S, unsigned ID); bool LookupName(LookupResult &R, Scope *S, bool AllowBuiltinCreation = false, @@ -7488,9 +7466,9 @@ public: bool LookupParsedName(LookupResult &R, Scope *S, CXXScopeSpec *SS, bool AllowBuiltinCreation = false, bool EnteringContext = false); - ObjCProtocolDecl * - LookupProtocol(IdentifierInfo *II, SourceLocation IdLoc, - RedeclarationKind Redecl = NotForRedeclaration); + ObjCProtocolDecl *LookupProtocol( + IdentifierInfo *II, SourceLocation IdLoc, + RedeclarationKind Redecl = RedeclarationKind::NotForRedeclaration); bool LookupInSuper(LookupResult &R, CXXRecordDecl *Class); void LookupOverloadedOperatorName(OverloadedOperatorKind Op, Scope *S, diff --git a/clang/lib/Interpreter/Interpreter.cpp b/clang/lib/Interpreter/Interpreter.cpp index cf31456b6950..b20e6efcebfd 100644 --- a/clang/lib/Interpreter/Interpreter.cpp +++ b/clang/lib/Interpreter/Interpreter.cpp @@ -550,7 +550,8 @@ std::unique_ptr Interpreter::FindRuntimeInterface() { auto LookupInterface = [&](Expr *&Interface, llvm::StringRef Name) { LookupResult R(S, &Ctx.Idents.get(Name), SourceLocation(), - Sema::LookupOrdinaryName, Sema::ForVisibleRedeclaration); + Sema::LookupOrdinaryName, + RedeclarationKind::ForVisibleRedeclaration); S.LookupQualifiedName(R, Ctx.getTranslationUnitDecl()); if (R.empty()) return false; diff --git a/clang/lib/Interpreter/InterpreterUtils.cpp b/clang/lib/Interpreter/InterpreterUtils.cpp index c19cf6aa3156..45f6322b8461 100644 --- a/clang/lib/Interpreter/InterpreterUtils.cpp +++ b/clang/lib/Interpreter/InterpreterUtils.cpp @@ -72,7 +72,7 @@ NamedDecl *LookupNamed(Sema &S, llvm::StringRef Name, const DeclContext *Within) { DeclarationName DName = &S.Context.Idents.get(Name); LookupResult R(S, DName, SourceLocation(), Sema::LookupOrdinaryName, - Sema::ForVisibleRedeclaration); + RedeclarationKind::ForVisibleRedeclaration); R.suppressDiagnostics(); diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp index 19abd5327b73..455ccb45b406 100644 --- a/clang/lib/Sema/SemaDecl.cpp +++ b/clang/lib/Sema/SemaDecl.cpp @@ -5374,7 +5374,7 @@ static bool CheckAnonMemberRedeclaration(Sema &SemaRef, Scope *S, LookupResult R(SemaRef, Name, NameLoc, Owner->isRecord() ? Sema::LookupMemberName : Sema::LookupOrdinaryName, - Sema::ForVisibleRedeclaration); + RedeclarationKind::ForVisibleRedeclaration); if (!SemaRef.LookupName(R, S)) return false; // Pick a representative declaration. @@ -6470,7 +6470,8 @@ NamedDecl *Sema::HandleDeclarator(Scope *S, Declarator &D, if (IsLinkageLookup) { Previous.clear(LookupRedeclarationWithLinkage); - Previous.setRedeclarationKind(ForExternalRedeclaration); + Previous.setRedeclarationKind( + RedeclarationKind::ForExternalRedeclaration); } LookupName(Previous, S, CreateBuiltins); @@ -8521,7 +8522,8 @@ void Sema::CheckShadow(Scope *S, VarDecl *D) { return; LookupResult R(*this, D->getDeclName(), D->getLocation(), - Sema::LookupOrdinaryName, Sema::ForVisibleRedeclaration); + Sema::LookupOrdinaryName, + RedeclarationKind::ForVisibleRedeclaration); LookupName(R, S); if (NamedDecl *ShadowedDecl = getShadowedDeclaration(D, R)) CheckShadow(D, ShadowedDecl, R); @@ -9161,7 +9163,7 @@ static NamedDecl *DiagnoseInvalidRedeclaration( LookupResult Prev(SemaRef, Name, NewFD->getLocation(), IsLocalFriend ? Sema::LookupLocalFriendName : Sema::LookupOrdinaryName, - Sema::ForVisibleRedeclaration); + RedeclarationKind::ForVisibleRedeclaration); NewFD->setInvalidDecl(); if (IsLocalFriend) @@ -15196,7 +15198,7 @@ Decl *Sema::ActOnParamDeclarator(Scope *S, Declarator &D, const IdentifierInfo *II = D.getIdentifier(); if (II) { LookupResult R(*this, II, D.getIdentifierLoc(), LookupOrdinaryName, - ForVisibleRedeclaration); + RedeclarationKind::ForVisibleRedeclaration); LookupName(R, S); if (!R.empty()) { NamedDecl *PrevDecl = *R.begin(); @@ -17428,7 +17430,7 @@ Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK, SourceLocation KWLoc, RedeclarationKind Redecl = forRedeclarationInCurContext(); if (TUK == TUK_Friend || TUK == TUK_Reference) - Redecl = NotForRedeclaration; + Redecl = RedeclarationKind::NotForRedeclaration; /// Create a new tag decl in C/ObjC. Since the ODR-like semantics for ObjC/C /// implemented asks for structural equivalence checking, the returned decl @@ -18589,7 +18591,7 @@ FieldDecl *Sema::HandleField(Scope *S, RecordDecl *Record, // Check to see if this name was declared as a member previously NamedDecl *PrevDecl = nullptr; LookupResult Previous(*this, II, Loc, LookupMemberName, - ForVisibleRedeclaration); + RedeclarationKind::ForVisibleRedeclaration); LookupName(Previous, S); switch (Previous.getResultKind()) { case LookupResult::Found: @@ -18993,8 +18995,9 @@ Decl *Sema::ActOnIvar(Scope *S, SourceLocation DeclStart, Declarator &D, NewID->setInvalidDecl(); if (II) { - NamedDecl *PrevDecl = LookupSingleName(S, II, Loc, LookupMemberName, - ForVisibleRedeclaration); + NamedDecl *PrevDecl = + LookupSingleName(S, II, Loc, LookupMemberName, + RedeclarationKind::ForVisibleRedeclaration); if (PrevDecl && isDeclInScope(PrevDecl, EnclosingContext, S) && !isa(PrevDecl)) { Diag(Loc, diag::err_duplicate_member) << II; @@ -20039,7 +20042,8 @@ Decl *Sema::ActOnEnumConstant(Scope *S, Decl *theEnumDecl, Decl *lastEnumConst, // Verify that there isn't already something declared with this name in this // scope. - LookupResult R(*this, Id, IdLoc, LookupOrdinaryName, ForVisibleRedeclaration); + LookupResult R(*this, Id, IdLoc, LookupOrdinaryName, + RedeclarationKind::ForVisibleRedeclaration); LookupName(R, S); NamedDecl *PrevDecl = R.getAsSingle(); diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp index 8c6bae545bfd..591016243b0a 100644 --- a/clang/lib/Sema/SemaDeclCXX.cpp +++ b/clang/lib/Sema/SemaDeclCXX.cpp @@ -896,7 +896,7 @@ Sema::ActOnDecompositionDeclarator(Scope *S, Declarator &D, assert(VarName && "Cannot have an unnamed binding declaration"); LookupResult Previous(*this, NameInfo, LookupOrdinaryName, - ForVisibleRedeclaration); + RedeclarationKind::ForVisibleRedeclaration); LookupName(Previous, S, /*CreateBuiltins*/DC->getRedeclContext()->isTranslationUnit()); @@ -951,7 +951,7 @@ Sema::ActOnDecompositionDeclarator(Scope *S, Declarator &D, DeclarationNameInfo NameInfo((IdentifierInfo *)nullptr, Decomp.getLSquareLoc()); LookupResult Previous(*this, NameInfo, LookupOrdinaryName, - ForVisibleRedeclaration); + RedeclarationKind::ForVisibleRedeclaration); // Build the variable that holds the non-decomposed object. bool AddToScope = true; @@ -11715,7 +11715,7 @@ Decl *Sema::ActOnStartNamespaceDef(Scope *NamespcScope, // look through using directives, just look for any ordinary names // as if by qualified name lookup. LookupResult R(*this, II, IdentLoc, LookupOrdinaryName, - ForExternalRedeclaration); + RedeclarationKind::ForExternalRedeclaration); LookupQualifiedName(R, CurContext->getRedeclContext()); NamedDecl *PrevDecl = R.isSingleResult() ? R.getRepresentativeDecl() : nullptr; @@ -12916,7 +12916,7 @@ NamedDecl *Sema::BuildUsingDeclaration( // Do the redeclaration lookup in the current scope. LookupResult Previous(*this, UsingName, LookupUsingDeclName, - ForVisibleRedeclaration); + RedeclarationKind::ForVisibleRedeclaration); Previous.setHideTags(false); if (S) { LookupName(Previous, S); @@ -13159,7 +13159,7 @@ NamedDecl *Sema::BuildUsingEnumDeclaration(Scope *S, AccessSpecifier AS, /// In class scope, check if this is a duplicate, for better a diagnostic. DeclarationNameInfo UsingEnumName(ED->getDeclName(), NameLoc); LookupResult Previous(*this, UsingEnumName, LookupUsingDeclName, - ForVisibleRedeclaration); + RedeclarationKind::ForVisibleRedeclaration); LookupName(Previous, S); @@ -13192,7 +13192,7 @@ NamedDecl *Sema::BuildUsingEnumDeclaration(Scope *S, AccessSpecifier AS, UsingShadowDecl *PrevDecl = nullptr; DeclarationNameInfo DNI(EC->getDeclName(), EC->getLocation()); LookupResult Previous(*this, DNI, LookupOrdinaryName, - ForVisibleRedeclaration); + RedeclarationKind::ForVisibleRedeclaration); LookupName(Previous, S); FilterUsingLookup(S, Previous); @@ -13587,7 +13587,7 @@ Decl *Sema::ActOnAliasDeclaration(Scope *S, AccessSpecifier AS, LookupResult Previous(*this, NameInfo, LookupOrdinaryName, TemplateParamLists.size() ? forRedeclarationInCurContext() - : ForVisibleRedeclaration); + : RedeclarationKind::ForVisibleRedeclaration); LookupName(Previous, S); // Warn about shadowing the name of a template parameter. @@ -13737,7 +13737,7 @@ Decl *Sema::ActOnNamespaceAliasDef(Scope *S, SourceLocation NamespaceLoc, // Check if we have a previous declaration with the same name. LookupResult PrevR(*this, Alias, AliasLoc, LookupOrdinaryName, - ForVisibleRedeclaration); + RedeclarationKind::ForVisibleRedeclaration); LookupName(PrevR, S); // Check we're not shadowing a template parameter. @@ -13983,7 +13983,7 @@ void Sema::CheckImplicitSpecialMemberDeclaration(Scope *S, FunctionDecl *FD) { // implicit special members with this name. DeclarationName Name = FD->getDeclName(); LookupResult R(*this, Name, SourceLocation(), LookupOrdinaryName, - ForExternalRedeclaration); + RedeclarationKind::ForExternalRedeclaration); for (auto *D : FD->getParent()->lookup(Name)) if (auto *Acceptable = R.getAcceptableDecl(D)) R.addDecl(Acceptable); @@ -17113,9 +17113,9 @@ Decl *Sema::ActOnExceptionDeclarator(Scope *S, Declarator &D) { } const IdentifierInfo *II = D.getIdentifier(); - if (NamedDecl *PrevDecl = LookupSingleName(S, II, D.getIdentifierLoc(), - LookupOrdinaryName, - ForVisibleRedeclaration)) { + if (NamedDecl *PrevDecl = + LookupSingleName(S, II, D.getIdentifierLoc(), LookupOrdinaryName, + RedeclarationKind::ForVisibleRedeclaration)) { // The scope should be freshly made just for us. There is just no way // it contains any previous declaration, except for function parameters in // a function-try-block's catch statement. @@ -17906,7 +17906,7 @@ NamedDecl *Sema::ActOnFriendFunctionDecl(Scope *S, Declarator &D, DeclContext *DC; Scope *DCScope = S; LookupResult Previous(*this, NameInfo, LookupOrdinaryName, - ForExternalRedeclaration); + RedeclarationKind::ForExternalRedeclaration); bool isTemplateId = D.getName().getKind() == UnqualifiedIdKind::IK_TemplateId; @@ -19242,7 +19242,7 @@ MSPropertyDecl *Sema::HandleMSProperty(Scope *S, RecordDecl *Record, // Check to see if this name was declared as a member previously NamedDecl *PrevDecl = nullptr; LookupResult Previous(*this, II, Loc, LookupMemberName, - ForVisibleRedeclaration); + RedeclarationKind::ForVisibleRedeclaration); LookupName(Previous, S); switch (Previous.getResultKind()) { case LookupResult::Found: diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp index f4a91ececfbb..7582cbd75fec 100644 --- a/clang/lib/Sema/SemaExprCXX.cpp +++ b/clang/lib/Sema/SemaExprCXX.cpp @@ -9153,7 +9153,7 @@ Sema::CheckMicrosoftIfExistsSymbol(Scope *S, // Do the redeclaration lookup in the current scope. LookupResult R(*this, TargetNameInfo, Sema::LookupAnyName, - Sema::NotForRedeclaration); + RedeclarationKind::NotForRedeclaration); LookupParsedName(R, S, &SS); R.suppressDiagnostics(); diff --git a/clang/lib/Sema/SemaExprMember.cpp b/clang/lib/Sema/SemaExprMember.cpp index 7ea6d733fe5a..c79128bc8f39 100644 --- a/clang/lib/Sema/SemaExprMember.cpp +++ b/clang/lib/Sema/SemaExprMember.cpp @@ -728,7 +728,7 @@ static bool LookupMemberExprInRecord(Sema &SemaRef, LookupResult &R, Sema &SemaRef; DeclarationNameInfo NameInfo; Sema::LookupNameKind LookupKind; - Sema::RedeclarationKind Redecl; + RedeclarationKind Redecl; }; QueryState Q = {R.getSema(), R.getLookupNameInfo(), R.getLookupKind(), R.redeclarationKind()}; diff --git a/clang/lib/Sema/SemaLookup.cpp b/clang/lib/Sema/SemaLookup.cpp index d65f52b8efe8..55af414df39f 100644 --- a/clang/lib/Sema/SemaLookup.cpp +++ b/clang/lib/Sema/SemaLookup.cpp @@ -4449,7 +4449,8 @@ LabelDecl *Sema::LookupOrCreateLabel(IdentifierInfo *II, SourceLocation Loc, } // Not a GNU local label. - Res = LookupSingleName(CurScope, II, Loc, LookupLabel, NotForRedeclaration); + Res = LookupSingleName(CurScope, II, Loc, LookupLabel, + RedeclarationKind::NotForRedeclaration); // If we found a label, check to see if it is in the same context as us. // When in a Block, we don't want to reuse a label in an enclosing function. if (Res && Res->getDeclContext() != CurContext) @@ -5889,7 +5890,8 @@ void Sema::clearDelayedTypo(TypoExpr *TE) { void Sema::ActOnPragmaDump(Scope *S, SourceLocation IILoc, IdentifierInfo *II) { DeclarationNameInfo Name(II, IILoc); - LookupResult R(*this, Name, LookupAnyName, Sema::NotForRedeclaration); + LookupResult R(*this, Name, LookupAnyName, + RedeclarationKind::NotForRedeclaration); R.suppressDiagnostics(); R.setHideTags(false); LookupName(R, S); @@ -5899,3 +5901,13 @@ void Sema::ActOnPragmaDump(Scope *S, SourceLocation IILoc, IdentifierInfo *II) { void Sema::ActOnPragmaDump(Expr *E) { E->dump(); } + +RedeclarationKind Sema::forRedeclarationInCurContext() const { + // A declaration with an owning module for linkage can never link against + // anything that is not visible. We don't need to check linkage here; if + // the context has internal linkage, redeclaration lookup won't find things + // from other TUs, and we can't safely compute linkage yet in general. + if (cast(CurContext)->getOwningModuleForLinkage(/*IgnoreLinkage*/ true)) + return RedeclarationKind::ForVisibleRedeclaration; + return RedeclarationKind::ForExternalRedeclaration; +} diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp index d229ef650bcc..3e9f6cba2507 100644 --- a/clang/lib/Sema/SemaOpenMP.cpp +++ b/clang/lib/Sema/SemaOpenMP.cpp @@ -24944,7 +24944,7 @@ ExprResult SemaOpenMP::ActOnOMPIteratorExpr(Scope *S, // Check for conflicting previous declaration. DeclarationNameInfo NameInfo(VD->getDeclName(), D.DeclIdentLoc); LookupResult Previous(SemaRef, NameInfo, Sema::LookupOrdinaryName, - Sema::ForVisibleRedeclaration); + RedeclarationKind::ForVisibleRedeclaration); Previous.suppressDiagnostics(); SemaRef.LookupName(Previous, S); diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp index 95171359f0ab..f4b6e1ceb6f0 100644 --- a/clang/lib/Sema/SemaTemplate.cpp +++ b/clang/lib/Sema/SemaTemplate.cpp @@ -972,8 +972,9 @@ void Sema::translateTemplateArguments(const ASTTemplateArgsPtr &TemplateArgsIn, static void maybeDiagnoseTemplateParameterShadow(Sema &SemaRef, Scope *S, SourceLocation Loc, const IdentifierInfo *Name) { - NamedDecl *PrevDecl = SemaRef.LookupSingleName( - S, Name, Loc, Sema::LookupOrdinaryName, Sema::ForVisibleRedeclaration); + NamedDecl *PrevDecl = + SemaRef.LookupSingleName(S, Name, Loc, Sema::LookupOrdinaryName, + RedeclarationKind::ForVisibleRedeclaration); if (PrevDecl && PrevDecl->isTemplateParameter()) SemaRef.DiagnoseTemplateParameterShadow(Loc, PrevDecl); } diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp index 6d359c5a9a02..caa07abb61fe 100644 --- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp +++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp @@ -2296,7 +2296,7 @@ Decl *TemplateDeclInstantiator::VisitFunctionDecl( SemaRef, Function->getDeclName(), SourceLocation(), D->isLocalExternDecl() ? Sema::LookupRedeclarationWithLinkage : Sema::LookupOrdinaryName, - D->isLocalExternDecl() ? Sema::ForExternalRedeclaration + D->isLocalExternDecl() ? RedeclarationKind::ForExternalRedeclaration : SemaRef.forRedeclarationInCurContext()); if (DependentFunctionTemplateSpecializationInfo *DFTSI = @@ -2697,7 +2697,7 @@ Decl *TemplateDeclInstantiator::VisitCXXMethodDecl( Method->setInvalidDecl(); LookupResult Previous(SemaRef, NameInfo, Sema::LookupOrdinaryName, - Sema::ForExternalRedeclaration); + RedeclarationKind::ForExternalRedeclaration); bool IsExplicitSpecialization = false; @@ -3365,7 +3365,7 @@ Decl *TemplateDeclInstantiator::VisitUsingDecl(UsingDecl *D) { // fact, it's not really even possible in non-class scopes). bool CheckRedeclaration = Owner->isRecord(); LookupResult Prev(SemaRef, NameInfo, Sema::LookupUsingDeclName, - Sema::ForVisibleRedeclaration); + RedeclarationKind::ForVisibleRedeclaration); UsingDecl *NewUD = UsingDecl::Create(SemaRef.Context, Owner, D->getUsingLoc(), @@ -5388,7 +5388,7 @@ void Sema::BuildVariableInstantiation( *this, NewVar->getDeclName(), NewVar->getLocation(), NewVar->isLocalExternDecl() ? Sema::LookupRedeclarationWithLinkage : Sema::LookupOrdinaryName, - NewVar->isLocalExternDecl() ? Sema::ForExternalRedeclaration + NewVar->isLocalExternDecl() ? RedeclarationKind::ForExternalRedeclaration : forRedeclarationInCurContext()); if (NewVar->isLocalExternDecl() && OldVar->getPreviousDecl() && -- GitLab From 950bb097e11d6ee26533c00519c62df994322228 Mon Sep 17 00:00:00 2001 From: Dinar Temirbulatov Date: Wed, 17 Apr 2024 15:30:40 +0000 Subject: [PATCH 065/862] Revert "[Clang][AArch64] Warn when calling non/streaming about vector size difference (#79842)" This reverts commit 4e85e1ffcaf161736e27a24c291c1177be865976 --- clang/include/clang/Basic/DiagnosticGroups.td | 3 - .../clang/Basic/DiagnosticSemaKinds.td | 10 -- clang/lib/Sema/SemaChecking.cpp | 22 +-- clang/lib/Sema/SemaDecl.cpp | 12 +- .../Sema/aarch64-incompat-sm-builtin-calls.c | 6 +- clang/test/Sema/aarch64-sme-func-attrs.c | 136 +----------------- 6 files changed, 5 insertions(+), 184 deletions(-) diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td index 47747d8704b6..5251774ff4ef 100644 --- a/clang/include/clang/Basic/DiagnosticGroups.td +++ b/clang/include/clang/Basic/DiagnosticGroups.td @@ -1412,9 +1412,6 @@ def MultiGPU: DiagGroup<"multi-gpu">; // libc and the CRT to be skipped. def AVRRtlibLinkingQuirks : DiagGroup<"avr-rtlib-linking-quirks">; -// A warning group related to AArch64 SME function attribues. -def AArch64SMEAttributes : DiagGroup<"aarch64-sme-attributes">; - // A warning group for things that will change semantics in the future. def FutureCompat : DiagGroup<"future-compat">; diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td index 44f802c0c28e..30a8543489f4 100644 --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -3751,16 +3751,6 @@ def err_sme_definition_using_za_in_non_sme_target : Error< "function using ZA state requires 'sme'">; def err_sme_definition_using_zt0_in_non_sme2_target : Error< "function using ZT0 state requires 'sme2'">; -def warn_sme_streaming_pass_return_vl_to_non_streaming : Warning< - "passing a VL-dependent argument to/from a function that has a different" - " streaming-mode. The streaming and non-streaming vector lengths may be" - " different">, - InGroup, DefaultIgnore; -def warn_sme_locally_streaming_has_vl_args_returns : Warning< - "passing/returning a VL-dependent argument to/from a __arm_locally_streaming" - " function. The streaming and non-streaming vector" - " lengths may be different">, - InGroup, DefaultIgnore; def err_conflicting_attributes_arm_state : Error< "conflicting attributes for state '%0'">; def err_sme_streaming_cannot_be_multiversioned : Error< diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp index 99b0a0008353..d814327d2347 100644 --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -7949,7 +7949,6 @@ void Sema::checkCall(NamedDecl *FDecl, const FunctionProtoType *Proto, // For variadic functions, we may have more args than parameters. // For some K&R functions, we may have less args than parameters. const auto N = std::min(Proto->getNumParams(), Args.size()); - bool AnyScalableArgsOrRet = Proto->getReturnType()->isSizelessVectorType(); for (unsigned ArgIdx = 0; ArgIdx < N; ++ArgIdx) { // Args[ArgIdx] can be null in malformed code. if (const Expr *Arg = Args[ArgIdx]) { @@ -7963,8 +7962,6 @@ void Sema::checkCall(NamedDecl *FDecl, const FunctionProtoType *Proto, checkAIXMemberAlignment((Arg->getExprLoc()), Arg); QualType ParamTy = Proto->getParamType(ArgIdx); - if (ParamTy->isSizelessVectorType()) - AnyScalableArgsOrRet = true; QualType ArgTy = Arg->getType(); CheckArgAlignment(Arg->getExprLoc(), FDecl, std::to_string(ArgIdx + 1), ArgTy, ParamTy); @@ -7985,23 +7982,6 @@ void Sema::checkCall(NamedDecl *FDecl, const FunctionProtoType *Proto, } } - // If the call requires a streaming-mode change and has scalable vector - // arguments or return values, then warn the user that the streaming and - // non-streaming vector lengths may be different. - const auto *CallerFD = dyn_cast(CurContext); - if (CallerFD && (!FD || !FD->getBuiltinID()) && AnyScalableArgsOrRet) { - bool IsCalleeStreaming = - ExtInfo.AArch64SMEAttributes & FunctionType::SME_PStateSMEnabledMask; - bool IsCalleeStreamingCompatible = - ExtInfo.AArch64SMEAttributes & - FunctionType::SME_PStateSMCompatibleMask; - ArmStreamingType CallerFnType = getArmStreamingFnType(CallerFD); - if (!IsCalleeStreamingCompatible && - (CallerFnType == ArmStreamingCompatible || - ((CallerFnType == ArmStreaming) ^ IsCalleeStreaming))) - Diag(Loc, diag::warn_sme_streaming_pass_return_vl_to_non_streaming); - } - FunctionType::ArmStateValue CalleeArmZAState = FunctionType::getArmZAState(ExtInfo.AArch64SMEAttributes); FunctionType::ArmStateValue CalleeArmZT0State = @@ -8010,7 +7990,7 @@ void Sema::checkCall(NamedDecl *FDecl, const FunctionProtoType *Proto, CalleeArmZT0State != FunctionType::ARM_None) { bool CallerHasZAState = false; bool CallerHasZT0State = false; - if (CallerFD) { + if (const auto *CallerFD = dyn_cast(CurContext)) { auto *Attr = CallerFD->getAttr(); if (Attr && Attr->isNewZA()) CallerHasZAState = true; diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp index 455ccb45b406..1bde99d6fce7 100644 --- a/clang/lib/Sema/SemaDecl.cpp +++ b/clang/lib/Sema/SemaDecl.cpp @@ -12408,22 +12408,12 @@ bool Sema::CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD, } // Check if the function definition uses any AArch64 SME features without - // having the '+sme' feature enabled and warn user if sme locally streaming - // function returns or uses arguments with VL-based types. + // having the '+sme' feature enabled. if (DeclIsDefn) { const auto *Attr = NewFD->getAttr(); bool UsesSM = NewFD->hasAttr(); bool UsesZA = Attr && Attr->isNewZA(); bool UsesZT0 = Attr && Attr->isNewZT0(); - - if (NewFD->hasAttr()) { - if (NewFD->getReturnType()->isSizelessVectorType() || - llvm::any_of(NewFD->parameters(), [](ParmVarDecl *P) { - return P->getOriginalType()->isSizelessVectorType(); - })) - Diag(NewFD->getLocation(), - diag::warn_sme_locally_streaming_has_vl_args_returns); - } if (const auto *FPT = NewFD->getType()->getAs()) { FunctionProtoType::ExtProtoInfo EPI = FPT->getExtProtoInfo(); UsesSM |= diff --git a/clang/test/Sema/aarch64-incompat-sm-builtin-calls.c b/clang/test/Sema/aarch64-incompat-sm-builtin-calls.c index 6a1feeb9bf53..55c97c73e8b6 100644 --- a/clang/test/Sema/aarch64-incompat-sm-builtin-calls.c +++ b/clang/test/Sema/aarch64-incompat-sm-builtin-calls.c @@ -1,6 +1,6 @@ // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve \ -// RUN: -target-feature +sme2 -target-feature +sve2 -target-feature +neon -Waarch64-sme-attributes -fsyntax-only -verify %s +// RUN: -target-feature +sme2 -target-feature +sve2 -target-feature +neon -fsyntax-only -verify %s // REQUIRES: aarch64-registered-target @@ -33,7 +33,6 @@ svuint32_t incompat_sve_sm(svbool_t pg, svuint32_t a, int16_t b) __arm_streaming return __builtin_sve_svld1_gather_u32base_index_u32(pg, a, b); } -// expected-warning@+1 {{passing/returning a VL-dependent argument to/from a __arm_locally_streaming function. The streaming and non-streaming vector lengths may be different}} __arm_locally_streaming svuint32_t incompat_sve_ls(svbool_t pg, svuint32_t a, int64_t b) { // expected-warning@+1 {{builtin call has undefined behaviour when called from a streaming function}} return __builtin_sve_svld1_gather_u32base_index_u32(pg, a, b); @@ -49,7 +48,6 @@ svuint32_t incompat_sve2_sm(svbool_t pg, svuint32_t a, int64_t b) __arm_streamin return __builtin_sve_svldnt1_gather_u32base_index_u32(pg, a, b); } -// expected-warning@+1 {{passing/returning a VL-dependent argument to/from a __arm_locally_streaming function. The streaming and non-streaming vector lengths may be different}} __arm_locally_streaming svuint32_t incompat_sve2_ls(svbool_t pg, svuint32_t a, int64_t b) { // expected-warning@+1 {{builtin call has undefined behaviour when called from a streaming function}} return __builtin_sve_svldnt1_gather_u32base_index_u32(pg, a, b); @@ -70,7 +68,6 @@ svfloat64_t streaming_caller_sve(svbool_t pg, svfloat64_t a, float64_t b) __arm_ return svadd_n_f64_m(pg, a, b); } -// expected-warning@+1 {{passing/returning a VL-dependent argument to/from a __arm_locally_streaming function. The streaming and non-streaming vector lengths may be different}} __arm_locally_streaming svfloat64_t locally_streaming_caller_sve(svbool_t pg, svfloat64_t a, float64_t b) { // expected-no-warning return svadd_n_f64_m(pg, a, b); @@ -86,7 +83,6 @@ svint16_t streaming_caller_sve2(svint16_t op1, svint16_t op2) __arm_streaming { return svmul_lane_s16(op1, op2, 0); } -// expected-warning@+1 {{passing/returning a VL-dependent argument to/from a __arm_locally_streaming function. The streaming and non-streaming vector lengths may be different}} __arm_locally_streaming svint16_t locally_streaming_caller_sve2(svint16_t op1, svint16_t op2) { // expected-no-warning return svmul_lane_s16(op1, op2, 0); diff --git a/clang/test/Sema/aarch64-sme-func-attrs.c b/clang/test/Sema/aarch64-sme-func-attrs.c index 12de16509ccb..bfc8768c3f36 100644 --- a/clang/test/Sema/aarch64-sme-func-attrs.c +++ b/clang/test/Sema/aarch64-sme-func-attrs.c @@ -1,5 +1,5 @@ -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sve -Waarch64-sme-attributes -fsyntax-only -verify %s -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sve -Waarch64-sme-attributes -fsyntax-only -verify=expected-cpp -x c++ %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -fsyntax-only -verify %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -fsyntax-only -verify=expected-cpp -x c++ %s // Valid attributes @@ -496,135 +496,3 @@ void fmv_caller() { just_fine(); incompatible_locally_streaming(); } - -void sme_streaming_with_vl_arg(__SVInt8_t a) __arm_streaming { } - -__SVInt8_t sme_streaming_returns_vl(void) __arm_streaming { __SVInt8_t r; return r; } - -void sme_streaming_compatible_with_vl_arg(__SVInt8_t a) __arm_streaming_compatible { } - -__SVInt8_t sme_streaming_compatible_returns_vl(void) __arm_streaming_compatible { __SVInt8_t r; return r; } - -void sme_no_streaming_with_vl_arg(__SVInt8_t a) { } - -__SVInt8_t sme_no_streaming_returns_vl(void) { __SVInt8_t r; return r; } - -// expected-warning@+2 {{passing/returning a VL-dependent argument to/from a __arm_locally_streaming function. The streaming and non-streaming vector lengths may be different}} -// expected-cpp-warning@+1 {{passing/returning a VL-dependent argument to/from a __arm_locally_streaming function. The streaming and non-streaming vector lengths may be different}} -__arm_locally_streaming void sme_locally_streaming_with_vl_arg(__SVInt8_t a) { } - -// expected-warning@+2 {{passing/returning a VL-dependent argument to/from a __arm_locally_streaming function. The streaming and non-streaming vector lengths may be different}} -// expected-cpp-warning@+1 {{passing/returning a VL-dependent argument to/from a __arm_locally_streaming function. The streaming and non-streaming vector lengths may be different}} -__arm_locally_streaming __SVInt8_t sme_locally_streaming_returns_vl(void) { __SVInt8_t r; return r; } - -void sme_no_streaming_calling_streaming_with_vl_args() { - __SVInt8_t a; - // expected-warning@+2 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - // expected-cpp-warning@+1 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - sme_streaming_with_vl_arg(a); -} - -void sme_no_streaming_calling_streaming_with_return_vl() { - // expected-warning@+2 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - // expected-cpp-warning@+1 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - __SVInt8_t r = sme_streaming_returns_vl(); -} - -void sme_streaming_calling_non_streaming_with_vl_args(void) __arm_streaming { - __SVInt8_t a; - // expected-warning@+2 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - // expected-cpp-warning@+1 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - sme_no_streaming_with_vl_arg(a); -} - -void sme_streaming_calling_non_streaming_with_return_vl(void) __arm_streaming { - // expected-warning@+2 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - // expected-cpp-warning@+1 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - __SVInt8_t r = sme_no_streaming_returns_vl(); -} - -void sme_no_streaming_calling_streaming_with_vl_args_param(__SVInt8_t arg, void (*sc)( __SVInt8_t arg) __arm_streaming) { - // expected-warning@+2 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - // expected-cpp-warning@+1 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - sc(arg); -} - -__SVInt8_t sme_no_streaming_calling_streaming_return_vl_param(__SVInt8_t (*s)(void) __arm_streaming) { - // expected-warning@+2 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - // expected-cpp-warning@+1 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - return s(); -} - -void sme_streaming_compatible_calling_streaming_with_vl_args(__SVInt8_t arg) __arm_streaming_compatible { - // expected-warning@+2 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - // expected-cpp-warning@+1 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - sme_streaming_with_vl_arg(arg); -} - -void sme_streaming_compatible_calling_sme_streaming_return_vl(void) __arm_streaming_compatible { - // expected-warning@+2 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - // expected-cpp-warning@+1 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - __SVInt8_t r = sme_streaming_returns_vl(); -} - -void sme_streaming_compatible_calling_no_streaming_with_vl_args(__SVInt8_t arg) __arm_streaming_compatible { - // expected-warning@+2 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - // expected-cpp-warning@+1 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - sme_no_streaming_with_vl_arg(arg); -} - -void sme_streaming_compatible_calling_no_sme_streaming_return_vl(void) __arm_streaming_compatible { - // expected-warning@+2 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - // expected-cpp-warning@+1 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - __SVInt8_t r = sme_no_streaming_returns_vl(); -} - -void sme_streaming_calling_streaming(__SVInt8_t arg, void (*s)( __SVInt8_t arg) __arm_streaming) __arm_streaming { - s(arg); -} - -__SVInt8_t sme_streaming_calling_streaming_return_vl(__SVInt8_t (*s)(void) __arm_streaming) __arm_streaming { - return s(); -} - -void sme_streaming_calling_streaming_with_vl_args(__SVInt8_t a) __arm_streaming { - sme_streaming_with_vl_arg(a); -} - -void sme_streaming_calling_streaming_with_return_vl(void) __arm_streaming { - __SVInt8_t r = sme_streaming_returns_vl(); -} - -void sme_streaming_calling_streaming_compatible_with_vl_args(__SVInt8_t a) __arm_streaming { - sme_streaming_compatible_with_vl_arg(a); -} - -void sme_streaming_calling_streaming_compatible_with_return_vl(void) __arm_streaming { - __SVInt8_t r = sme_streaming_compatible_returns_vl(); -} - -void sme_no_streaming_calling_streaming_compatible_with_vl_args() { - __SVInt8_t a; - sme_streaming_compatible_with_vl_arg(a); -} - -void sme_no_streaming_calling_streaming_compatible_with_return_vl() { - __SVInt8_t r = sme_streaming_compatible_returns_vl(); -} - -void sme_no_streaming_calling_non_streaming_compatible_with_vl_args() { - __SVInt8_t a; - sme_no_streaming_with_vl_arg(a); -} - -void sme_no_streaming_calling_non_streaming_compatible_with_return_vl() { - __SVInt8_t r = sme_no_streaming_returns_vl(); -} - -void sme_streaming_compatible_calling_streaming_compatible_with_vl_args(__SVInt8_t arg) __arm_streaming_compatible { - sme_streaming_compatible_with_vl_arg(arg); -} - -void sme_streaming_compatible_calling_streaming_compatible_with_return_vl(void) __arm_streaming_compatible { - __SVInt8_t r = sme_streaming_compatible_returns_vl(); -} -- GitLab From b854a2323337be2633b1135f590678a17e9d1ade Mon Sep 17 00:00:00 2001 From: Robin Caloudis Date: Wed, 17 Apr 2024 17:38:47 +0200 Subject: [PATCH 066/862] [libc][c23][fenv] Implement fetestexceptflag (#87828) Provide C23 `fetestexceptflag` function according to 7.6.4.6 in the latest [revision of the C standard](https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3096.pdf) from 2023-04-02. Closes https://github.com/llvm/llvm-project/issues/87565. --- libc/config/baremetal/arm/entrypoints.txt | 1 + libc/config/baremetal/riscv/entrypoints.txt | 1 + libc/config/darwin/arm/entrypoints.txt | 1 + libc/config/darwin/x86_64/entrypoints.txt | 1 + libc/config/linux/aarch64/entrypoints.txt | 1 + libc/config/linux/arm/entrypoints.txt | 1 + libc/config/linux/riscv/entrypoints.txt | 1 + libc/config/linux/x86_64/entrypoints.txt | 1 + libc/config/windows/entrypoints.txt | 1 + libc/docs/c23.rst | 2 +- libc/docs/fenv.rst | 4 +-- libc/spec/stdc.td | 5 +++ libc/src/fenv/CMakeLists.txt | 13 ++++++++ libc/src/fenv/fetestexceptflag.cpp | 23 +++++++++++++ libc/src/fenv/fetestexceptflag.h | 20 ++++++++++++ libc/test/src/fenv/CMakeLists.txt | 1 + libc/test/src/fenv/exception_flags_test.cpp | 32 +++++++++++++++---- .../llvm-project-overlay/libc/BUILD.bazel | 10 ++++++ .../libc/test/src/fenv/BUILD.bazel | 1 + 19 files changed, 110 insertions(+), 10 deletions(-) create mode 100644 libc/src/fenv/fetestexceptflag.cpp create mode 100644 libc/src/fenv/fetestexceptflag.h diff --git a/libc/config/baremetal/arm/entrypoints.txt b/libc/config/baremetal/arm/entrypoints.txt index f33f9430c792..4e3d1cb9f533 100644 --- a/libc/config/baremetal/arm/entrypoints.txt +++ b/libc/config/baremetal/arm/entrypoints.txt @@ -201,6 +201,7 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.fenv.fesetround libc.src.fenv.feraiseexcept libc.src.fenv.fetestexcept + libc.src.fenv.fetestexceptflag libc.src.fenv.feupdateenv # math.h entrypoints diff --git a/libc/config/baremetal/riscv/entrypoints.txt b/libc/config/baremetal/riscv/entrypoints.txt index dad187fa0496..7efd9bcd5b3c 100644 --- a/libc/config/baremetal/riscv/entrypoints.txt +++ b/libc/config/baremetal/riscv/entrypoints.txt @@ -201,6 +201,7 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.fenv.fesetround libc.src.fenv.feraiseexcept libc.src.fenv.fetestexcept + libc.src.fenv.fetestexceptflag libc.src.fenv.feupdateenv # math.h entrypoints diff --git a/libc/config/darwin/arm/entrypoints.txt b/libc/config/darwin/arm/entrypoints.txt index aea2f6d5771e..e1303265b9ac 100644 --- a/libc/config/darwin/arm/entrypoints.txt +++ b/libc/config/darwin/arm/entrypoints.txt @@ -112,6 +112,7 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.fenv.fesetround libc.src.fenv.feraiseexcept libc.src.fenv.fetestexcept + libc.src.fenv.fetestexceptflag libc.src.fenv.feupdateenv # math.h entrypoints diff --git a/libc/config/darwin/x86_64/entrypoints.txt b/libc/config/darwin/x86_64/entrypoints.txt index 09fe3d7b4768..02912decadcf 100644 --- a/libc/config/darwin/x86_64/entrypoints.txt +++ b/libc/config/darwin/x86_64/entrypoints.txt @@ -106,6 +106,7 @@ set(TARGET_LIBM_ENTRYPOINTS # libc.src.fenv.fesetround # libc.src.fenv.feraiseexcept # libc.src.fenv.fetestexcept + # libc.src.fenv.fetestexceptflag # libc.src.fenv.feupdateenv ## Currently disabled for failing tests. diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt index 2952baacdd67..1ac6bd930000 100644 --- a/libc/config/linux/aarch64/entrypoints.txt +++ b/libc/config/linux/aarch64/entrypoints.txt @@ -324,6 +324,7 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.fenv.fesetround libc.src.fenv.feraiseexcept libc.src.fenv.fetestexcept + libc.src.fenv.fetestexceptflag libc.src.fenv.feupdateenv # math.h entrypoints diff --git a/libc/config/linux/arm/entrypoints.txt b/libc/config/linux/arm/entrypoints.txt index 35fd588a9a6c..335981ff7dc7 100644 --- a/libc/config/linux/arm/entrypoints.txt +++ b/libc/config/linux/arm/entrypoints.txt @@ -192,6 +192,7 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.fenv.fesetround libc.src.fenv.feraiseexcept libc.src.fenv.fetestexcept + libc.src.fenv.fetestexceptflag libc.src.fenv.feupdateenv # math.h entrypoints diff --git a/libc/config/linux/riscv/entrypoints.txt b/libc/config/linux/riscv/entrypoints.txt index 47c03a61c45a..87e82e5eb9a0 100644 --- a/libc/config/linux/riscv/entrypoints.txt +++ b/libc/config/linux/riscv/entrypoints.txt @@ -332,6 +332,7 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.fenv.fesetround libc.src.fenv.feraiseexcept libc.src.fenv.fetestexcept + libc.src.fenv.fetestexceptflag libc.src.fenv.feupdateenv # math.h entrypoints diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt index 8fdd4575e27e..70f130a4399a 100644 --- a/libc/config/linux/x86_64/entrypoints.txt +++ b/libc/config/linux/x86_64/entrypoints.txt @@ -346,6 +346,7 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.fenv.fesetround libc.src.fenv.feraiseexcept libc.src.fenv.fetestexcept + libc.src.fenv.fetestexceptflag libc.src.fenv.feupdateenv # math.h entrypoints diff --git a/libc/config/windows/entrypoints.txt b/libc/config/windows/entrypoints.txt index c46c947bf313..71216530c404 100644 --- a/libc/config/windows/entrypoints.txt +++ b/libc/config/windows/entrypoints.txt @@ -110,6 +110,7 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.fenv.fesetround libc.src.fenv.feraiseexcept libc.src.fenv.fetestexcept + libc.src.fenv.fetestexceptflag libc.src.fenv.feupdateenv # math.h entrypoints diff --git a/libc/docs/c23.rst b/libc/docs/c23.rst index 4138c9d7104f..44724fe1660c 100644 --- a/libc/docs/c23.rst +++ b/libc/docs/c23.rst @@ -21,7 +21,7 @@ Additions: * fenv.h * fesetexcept |check| - * fetestexceptflag + * fetestexceptflag |check| * fegetmode * fesetmode * math.h diff --git a/libc/docs/fenv.rst b/libc/docs/fenv.rst index 6574fb7246dd..1dee5515e117 100644 --- a/libc/docs/fenv.rst +++ b/libc/docs/fenv.rst @@ -42,7 +42,7 @@ fenv.h Functions - |check| - 7.6.6.3 * - fesetexcept - - + - |check| - 7.6.4.4 * - fesetexceptflag - |check| @@ -57,7 +57,7 @@ fenv.h Functions - |check| - 7.6.4.7 * - fetestexceptflag - - + - |check| - 7.6.4.6 * - feupdateenv - |check| diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td index 63d044986711..01aa7c70b3b9 100644 --- a/libc/spec/stdc.td +++ b/libc/spec/stdc.td @@ -149,6 +149,11 @@ def StdC : StandardSpec<"stdc"> { RetValSpec, [ArgSpec] >, + FunctionSpec< + "fetestexceptflag", + RetValSpec, + [ArgSpec, ArgSpec] + >, FunctionSpec< "feraiseexcept", RetValSpec, diff --git a/libc/src/fenv/CMakeLists.txt b/libc/src/fenv/CMakeLists.txt index 17e994741206..c5431b1b9d55 100644 --- a/libc/src/fenv/CMakeLists.txt +++ b/libc/src/fenv/CMakeLists.txt @@ -58,6 +58,19 @@ add_entrypoint_object( -O2 ) +add_entrypoint_object( + fetestexceptflag + SRCS + fetestexceptflag.cpp + HDRS + fetestexceptflag.h + DEPENDS + libc.hdr.types.fexcept_t + libc.src.__support.FPUtil.fenv_impl + COMPILE_OPTIONS + -O2 +) + add_entrypoint_object( fegetenv SRCS diff --git a/libc/src/fenv/fetestexceptflag.cpp b/libc/src/fenv/fetestexceptflag.cpp new file mode 100644 index 000000000000..63453350a199 --- /dev/null +++ b/libc/src/fenv/fetestexceptflag.cpp @@ -0,0 +1,23 @@ +//===-- Implementation of fetestexceptflag function -----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/fenv/fetestexceptflag.h" +#include "hdr/types/fexcept_t.h" +#include "src/__support/FPUtil/FEnvImpl.h" +#include "src/__support/common.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(int, fetestexceptflag, + (const fexcept_t *flagp, int excepts)) { + static_assert(sizeof(int) >= sizeof(fexcept_t), + "fexcept_t value cannot fit in an int value."); + return *flagp | fputil::test_except(excepts); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/fenv/fetestexceptflag.h b/libc/src/fenv/fetestexceptflag.h new file mode 100644 index 000000000000..1c8b0b843f54 --- /dev/null +++ b/libc/src/fenv/fetestexceptflag.h @@ -0,0 +1,20 @@ +//===-- Implementation header for fetestexceptflag --------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_FENV_FETESTEXCEPTFLAG_H +#define LLVM_LIBC_SRC_FENV_FETESTEXCEPTFLAG_H + +#include "hdr/types/fexcept_t.h" + +namespace LIBC_NAMESPACE { + +int fetestexceptflag(const fexcept_t *, int excepts); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_FENV_FETESTEXCEPTFLAG_H diff --git a/libc/test/src/fenv/CMakeLists.txt b/libc/test/src/fenv/CMakeLists.txt index 577735599dc0..f277b65e2d42 100644 --- a/libc/test/src/fenv/CMakeLists.txt +++ b/libc/test/src/fenv/CMakeLists.txt @@ -48,6 +48,7 @@ add_libc_unittest( DEPENDS libc.src.fenv.fegetexceptflag libc.src.fenv.fesetexceptflag + libc.src.fenv.fetestexceptflag libc.src.__support.FPUtil.fenv_impl ) diff --git a/libc/test/src/fenv/exception_flags_test.cpp b/libc/test/src/fenv/exception_flags_test.cpp index d1d8bfcc53db..9d2be6426a6d 100644 --- a/libc/test/src/fenv/exception_flags_test.cpp +++ b/libc/test/src/fenv/exception_flags_test.cpp @@ -1,4 +1,4 @@ -//===-- Unittests for fegetexceptflag and fesetexceptflag -----------------===// +//===-- Unittests for fe{get|set|test}exceptflag --------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -9,11 +9,12 @@ #include "hdr/types/fexcept_t.h" #include "src/fenv/fegetexceptflag.h" #include "src/fenv/fesetexceptflag.h" +#include "src/fenv/fetestexceptflag.h" #include "src/__support/FPUtil/FEnvImpl.h" #include "test/UnitTest/Test.h" -TEST(LlvmLibcFenvTest, GetExceptFlagAndSetExceptFlag) { +TEST(LlvmLibcFenvTest, GetSetTestExceptFlag) { // We will disable all exceptions to prevent invocation of the exception // handler. LIBC_NAMESPACE::fputil::disable_except(FE_ALL_EXCEPT); @@ -39,19 +40,36 @@ TEST(LlvmLibcFenvTest, GetExceptFlagAndSetExceptFlag) { ASSERT_EQ(LIBC_NAMESPACE::fesetexceptflag(&eflags, FE_ALL_EXCEPT), 0); ASSERT_NE(LIBC_NAMESPACE::fputil::test_except(FE_ALL_EXCEPT) & e, 0); + // Exception flags are exactly the flags corresponding to the previously + // raised exception. + ASSERT_EQ(LIBC_NAMESPACE::fetestexceptflag(&eflags, FE_ALL_EXCEPT), + LIBC_NAMESPACE::fputil::test_except(FE_ALL_EXCEPT)); + // Cleanup. We clear all excepts as raising excepts like FE_OVERFLOW // can also raise FE_INEXACT. LIBC_NAMESPACE::fputil::clear_except(FE_ALL_EXCEPT); } - // Next, we will raise one exception and save the flags. + // Next, we will raise one exception, save the flag and clear all exceptions. LIBC_NAMESPACE::fputil::raise_except(FE_INVALID); - fexcept_t eflags; - LIBC_NAMESPACE::fegetexceptflag(&eflags, FE_ALL_EXCEPT); - // Clear all exceptions and raise two other exceptions. + fexcept_t invalid_flag; + LIBC_NAMESPACE::fegetexceptflag(&invalid_flag, FE_ALL_EXCEPT); + ASSERT_EQ(LIBC_NAMESPACE::fetestexceptflag(&invalid_flag, FE_ALL_EXCEPT), + FE_INVALID); LIBC_NAMESPACE::fputil::clear_except(FE_ALL_EXCEPT); + + // Raise two other exceptions and verify that they are set. LIBC_NAMESPACE::fputil::raise_except(FE_OVERFLOW | FE_INEXACT); + fexcept_t overflow_and_inexact_flag; + LIBC_NAMESPACE::fegetexceptflag(&overflow_and_inexact_flag, FE_ALL_EXCEPT); + ASSERT_EQ(LIBC_NAMESPACE::fetestexceptflag(&overflow_and_inexact_flag, + FE_ALL_EXCEPT), + FE_OVERFLOW | FE_INEXACT); + ASSERT_EQ(LIBC_NAMESPACE::fetestexceptflag(&overflow_and_inexact_flag, + FE_OVERFLOW | FE_INEXACT), + FE_OVERFLOW | FE_INEXACT); + // When we set the flags and test, we should only see FE_INVALID. - LIBC_NAMESPACE::fesetexceptflag(&eflags, FE_ALL_EXCEPT); + LIBC_NAMESPACE::fesetexceptflag(&invalid_flag, FE_ALL_EXCEPT); EXPECT_EQ(LIBC_NAMESPACE::fputil::test_except(FE_ALL_EXCEPT), FE_INVALID); } diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel index fb37f113b310..6029cc3fee61 100644 --- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel @@ -1145,6 +1145,16 @@ libc_function( ], ) +libc_function( + name = "fetestexceptflag", + srcs = ["src/fenv/fetestexceptflag.cpp"], + hdrs = ["src/fenv/fetestexceptflag.h"], + deps = [ + ":__support_common", + ":__support_fputil_fenv_impl", + ], +) + libc_function( name = "feclearexcept", srcs = ["src/fenv/feclearexcept.cpp"], diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/fenv/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/fenv/BUILD.bazel index 359db0723dfd..fc3ab3da3587 100644 --- a/utils/bazel/llvm-project-overlay/libc/test/src/fenv/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/test/src/fenv/BUILD.bazel @@ -75,6 +75,7 @@ libc_test( libc_function_deps = [ "//libc:fegetexceptflag", "//libc:fesetexceptflag", + "//libc:fetestexceptflag", ], deps = [ "//libc:__support_fputil_fenv_impl", -- GitLab From 8656d4c6a7a742c6fa6ee02c2ace7415163e65e4 Mon Sep 17 00:00:00 2001 From: Krystian Stasiowski Date: Wed, 17 Apr 2024 11:41:03 -0400 Subject: [PATCH 067/862] [Clang][Parse] Diagnose requires expressions with explicit object parameters (#88974) Clang currently allows the following: ``` auto x = requires (this int) { true; }; ``` This patch addresses that. --- clang/docs/ReleaseNotes.rst | 2 ++ clang/include/clang/Basic/DiagnosticParseKinds.td | 2 ++ clang/lib/Parse/ParseDecl.cpp | 15 ++++++++++++++- .../CXX/dcl.decl/dcl.meaning/dcl.fct/p6-cxx23.cpp | 7 +++++++ 4 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 clang/test/CXX/dcl.decl/dcl.meaning/dcl.fct/p6-cxx23.cpp diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 96ad92b540b4..c19ad9fba58f 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -364,6 +364,8 @@ Improvements to Clang's diagnostics - Clang now uses the correct type-parameter-key (``class`` or ``typename``) when printing template template parameter declarations. +- Clang now diagnoses requires expressions with explicit object parameters. + Improvements to Clang's time-trace ---------------------------------- diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td index bb9ca2a50cc0..66405095d51d 100644 --- a/clang/include/clang/Basic/DiagnosticParseKinds.td +++ b/clang/include/clang/Basic/DiagnosticParseKinds.td @@ -863,6 +863,8 @@ def err_empty_requires_expr : Error< "a requires expression must contain at least one requirement">; def err_requires_expr_parameter_list_ellipsis : Error< "varargs not allowed in requires expression">; +def err_requires_expr_explicit_object_parameter: Error< + "a requires expression cannot have an explicit object parameter">; def err_expected_semi_requirement : Error< "expected ';' at end of requirement">; def err_requires_expr_missing_arrow : Error< diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp index 274ee7b10c17..5f26b5a9e46b 100644 --- a/clang/lib/Parse/ParseDecl.cpp +++ b/clang/lib/Parse/ParseDecl.cpp @@ -7660,8 +7660,21 @@ void Parser::ParseParameterDeclarationClause( // Parse a C++23 Explicit Object Parameter // We do that in all language modes to produce a better diagnostic. SourceLocation ThisLoc; - if (getLangOpts().CPlusPlus && Tok.is(tok::kw_this)) + if (getLangOpts().CPlusPlus && Tok.is(tok::kw_this)) { ThisLoc = ConsumeToken(); + // C++23 [dcl.fct]p6: + // An explicit-object-parameter-declaration is a parameter-declaration + // with a this specifier. An explicit-object-parameter-declaration + // shall appear only as the first parameter-declaration of a + // parameter-declaration-list of either: + // - a member-declarator that declares a member function, or + // - a lambda-declarator. + // + // The parameter-declaration-list of a requires-expression is not such + // a context. + if (DeclaratorCtx == DeclaratorContext::RequiresExpr) + Diag(ThisLoc, diag::err_requires_expr_explicit_object_parameter); + } ParseDeclarationSpecifiers(DS, /*TemplateInfo=*/ParsedTemplateInfo(), AS_none, DeclSpecContext::DSC_normal, diff --git a/clang/test/CXX/dcl.decl/dcl.meaning/dcl.fct/p6-cxx23.cpp b/clang/test/CXX/dcl.decl/dcl.meaning/dcl.fct/p6-cxx23.cpp new file mode 100644 index 000000000000..9c1f30f81a01 --- /dev/null +++ b/clang/test/CXX/dcl.decl/dcl.meaning/dcl.fct/p6-cxx23.cpp @@ -0,0 +1,7 @@ +// RUN: %clang_cc1 -std=c++23 -fsyntax-only -verify %s + +auto x0 = requires (this int) { true; }; // expected-error {{a requires expression cannot have an explicit object parameter}} +auto x1 = requires (int, this int) { true; }; // expected-error {{a requires expression cannot have an explicit object parameter}} + +template // expected-error {{expected template parameter}} +void f(); // expected-error {{no function template matches function template specialization 'f'}} -- GitLab From abd5e45a96954d80f6ffe6d8676c0059fae8573b Mon Sep 17 00:00:00 2001 From: Alexander Richardson Date: Wed, 17 Apr 2024 08:42:41 -0700 Subject: [PATCH 068/862] [compiler-rt] Use __atomic builtins whenever possible The code in this file dates back to 2012 when Clang's support for atomic builtins was still quite limited. The bugs referenced in the comment at the top of the file have long been fixed and using the compiler builtins directly should now generate slightly better code. Additionally, this allows using the atomic builtin header for platforms where the __sync_builtins are lacking (e.g. Arm Morello). This change does not introduce any code generation changes for __tsan_read*/__tsan_write* or __tsan_func_{entry,exit} on x86, which indicates the previously noted compiler issues have been fixed. We also have to touch the non-clang codepaths here since the only way we can make this work easily is by making the memory_order enum match the compiler-provided macros, so we have to update the debug checks that assumed the enum was always a bitflag. The one downside of this change is that 32-bit MIPS now definitely requires libatomic (but that may already have been needed for RMW ops). Reviewed By: dvyukov Pull Request: https://github.com/llvm/llvm-project/pull/84439 --- .../lib/sanitizer_common/CMakeLists.txt | 3 - .../lib/sanitizer_common/sanitizer_atomic.h | 12 ++ .../sanitizer_common/sanitizer_atomic_clang.h | 85 ++++++------- .../sanitizer_atomic_clang_mips.h | 117 ------------------ .../sanitizer_atomic_clang_other.h | 85 ------------- .../sanitizer_atomic_clang_x86.h | 113 ----------------- .../sanitizer_common/sanitizer_atomic_msvc.h | 8 +- .../compiler-rt/lib/sanitizer_common/BUILD.gn | 3 - 8 files changed, 56 insertions(+), 370 deletions(-) delete mode 100644 compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_mips.h delete mode 100644 compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_other.h delete mode 100644 compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_x86.h diff --git a/compiler-rt/lib/sanitizer_common/CMakeLists.txt b/compiler-rt/lib/sanitizer_common/CMakeLists.txt index f2b4ac72ae15..66f2d259aa5f 100644 --- a/compiler-rt/lib/sanitizer_common/CMakeLists.txt +++ b/compiler-rt/lib/sanitizer_common/CMakeLists.txt @@ -122,9 +122,6 @@ set(SANITIZER_IMPL_HEADERS sanitizer_asm.h sanitizer_atomic.h sanitizer_atomic_clang.h - sanitizer_atomic_clang_mips.h - sanitizer_atomic_clang_other.h - sanitizer_atomic_clang_x86.h sanitizer_atomic_msvc.h sanitizer_bitvector.h sanitizer_bvgraph.h diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_atomic.h b/compiler-rt/lib/sanitizer_common/sanitizer_atomic.h index 46f06957228c..0609a11ffdeb 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_atomic.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_atomic.h @@ -18,12 +18,24 @@ namespace __sanitizer { enum memory_order { +// If the __atomic atomic builtins are supported (Clang/GCC), use the +// compiler provided macro values so that we can map the atomic operations +// to __atomic_* directly. +#ifdef __ATOMIC_SEQ_CST + memory_order_relaxed = __ATOMIC_RELAXED, + memory_order_consume = __ATOMIC_CONSUME, + memory_order_acquire = __ATOMIC_ACQUIRE, + memory_order_release = __ATOMIC_RELEASE, + memory_order_acq_rel = __ATOMIC_ACQ_REL, + memory_order_seq_cst = __ATOMIC_SEQ_CST +#else memory_order_relaxed = 1 << 0, memory_order_consume = 1 << 1, memory_order_acquire = 1 << 2, memory_order_release = 1 << 3, memory_order_acq_rel = 1 << 4, memory_order_seq_cst = 1 << 5 +#endif }; struct atomic_uint8_t { diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang.h b/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang.h index 4318d64d16cf..1414092e38d7 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang.h @@ -14,60 +14,63 @@ #ifndef SANITIZER_ATOMIC_CLANG_H #define SANITIZER_ATOMIC_CLANG_H -#if defined(__i386__) || defined(__x86_64__) -# include "sanitizer_atomic_clang_x86.h" -#else -# include "sanitizer_atomic_clang_other.h" -#endif - namespace __sanitizer { -// We would like to just use compiler builtin atomic operations -// for loads and stores, but they are mostly broken in clang: -// - they lead to vastly inefficient code generation -// (http://llvm.org/bugs/show_bug.cgi?id=17281) -// - 64-bit atomic operations are not implemented on x86_32 -// (http://llvm.org/bugs/show_bug.cgi?id=15034) -// - they are not implemented on ARM -// error: undefined reference to '__atomic_load_4' +// We use the compiler builtin atomic operations for loads and stores, which +// generates correct code for all architectures, but may require libatomic +// on platforms where e.g. 64-bit atomics are not supported natively. // See http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html // for mappings of the memory model to different processors. -inline void atomic_signal_fence(memory_order) { +inline void atomic_signal_fence(memory_order mo) { __atomic_signal_fence(mo); } + +inline void atomic_thread_fence(memory_order mo) { __atomic_thread_fence(mo); } + +inline void proc_yield(int cnt) { + __asm__ __volatile__("" ::: "memory"); +#if defined(__i386__) || defined(__x86_64__) + for (int i = 0; i < cnt; i++) __asm__ __volatile__("pause"); __asm__ __volatile__("" ::: "memory"); +#endif } -inline void atomic_thread_fence(memory_order) { - __sync_synchronize(); +template +inline typename T::Type atomic_load(const volatile T *a, memory_order mo) { + DCHECK(mo == memory_order_relaxed || mo == memory_order_consume || + mo == memory_order_acquire || mo == memory_order_seq_cst); + DCHECK(!((uptr)a % sizeof(*a))); + return __atomic_load_n(&a->val_dont_use, mo); } -template -inline typename T::Type atomic_fetch_add(volatile T *a, - typename T::Type v, memory_order mo) { - (void)mo; +template +inline void atomic_store(volatile T *a, typename T::Type v, memory_order mo) { + DCHECK(mo == memory_order_relaxed || mo == memory_order_release || + mo == memory_order_seq_cst); DCHECK(!((uptr)a % sizeof(*a))); - return __sync_fetch_and_add(&a->val_dont_use, v); + __atomic_store_n(&a->val_dont_use, v, mo); } -template -inline typename T::Type atomic_fetch_sub(volatile T *a, - typename T::Type v, memory_order mo) { +template +inline typename T::Type atomic_fetch_add(volatile T *a, typename T::Type v, + memory_order mo) { + DCHECK(!((uptr)a % sizeof(*a))); + return __atomic_fetch_add(&a->val_dont_use, v, mo); +} + +template +inline typename T::Type atomic_fetch_sub(volatile T *a, typename T::Type v, + memory_order mo) { (void)mo; DCHECK(!((uptr)a % sizeof(*a))); - return __sync_fetch_and_add(&a->val_dont_use, -v); + return __atomic_fetch_sub(&a->val_dont_use, v, mo); } -template -inline typename T::Type atomic_exchange(volatile T *a, - typename T::Type v, memory_order mo) { +template +inline typename T::Type atomic_exchange(volatile T *a, typename T::Type v, + memory_order mo) { DCHECK(!((uptr)a % sizeof(*a))); - if (mo & (memory_order_release | memory_order_acq_rel | memory_order_seq_cst)) - __sync_synchronize(); - v = __sync_lock_test_and_set(&a->val_dont_use, v); - if (mo == memory_order_seq_cst) - __sync_synchronize(); - return v; + return __atomic_exchange_n(&a->val_dont_use, v, mo); } template @@ -82,9 +85,8 @@ inline bool atomic_compare_exchange_strong(volatile T *a, typename T::Type *cmp, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } -template -inline bool atomic_compare_exchange_weak(volatile T *a, - typename T::Type *cmp, +template +inline bool atomic_compare_exchange_weak(volatile T *a, typename T::Type *cmp, typename T::Type xchg, memory_order mo) { return atomic_compare_exchange_strong(a, cmp, xchg, mo); @@ -92,13 +94,6 @@ inline bool atomic_compare_exchange_weak(volatile T *a, } // namespace __sanitizer -// This include provides explicit template instantiations for atomic_uint64_t -// on MIPS32, which does not directly support 8 byte atomics. It has to -// proceed the template definitions above. -#if defined(_MIPS_SIM) && defined(_ABIO32) && _MIPS_SIM == _ABIO32 -# include "sanitizer_atomic_clang_mips.h" -#endif - #undef ATOMIC_ORDER #endif // SANITIZER_ATOMIC_CLANG_H diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_mips.h b/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_mips.h deleted file mode 100644 index f3d3052e5b7c..000000000000 --- a/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_mips.h +++ /dev/null @@ -1,117 +0,0 @@ -//===-- sanitizer_atomic_clang_mips.h ---------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This file is a part of ThreadSanitizer/AddressSanitizer runtime. -// Not intended for direct inclusion. Include sanitizer_atomic.h. -// -//===----------------------------------------------------------------------===// - -#ifndef SANITIZER_ATOMIC_CLANG_MIPS_H -#define SANITIZER_ATOMIC_CLANG_MIPS_H - -namespace __sanitizer { - -// MIPS32 does not support atomics > 4 bytes. To address this lack of -// functionality, the sanitizer library provides helper methods which use an -// internal spin lock mechanism to emulate atomic operations when the size is -// 8 bytes. -static void __spin_lock(volatile int *lock) { - while (__sync_lock_test_and_set(lock, 1)) - while (*lock) { - } -} - -static void __spin_unlock(volatile int *lock) { __sync_lock_release(lock); } - -// Make sure the lock is on its own cache line to prevent false sharing. -// Put it inside a struct that is aligned and padded to the typical MIPS -// cacheline which is 32 bytes. -static struct { - int lock; - char pad[32 - sizeof(int)]; -} __attribute__((aligned(32))) lock = {0, {0}}; - -template <> -inline atomic_uint64_t::Type atomic_fetch_add(volatile atomic_uint64_t *ptr, - atomic_uint64_t::Type val, - memory_order mo) { - DCHECK(mo & - (memory_order_relaxed | memory_order_release | memory_order_seq_cst)); - DCHECK(!((uptr)ptr % sizeof(*ptr))); - - atomic_uint64_t::Type ret; - - __spin_lock(&lock.lock); - ret = *(const_cast(&ptr->val_dont_use)); - ptr->val_dont_use = ret + val; - __spin_unlock(&lock.lock); - - return ret; -} - -template <> -inline atomic_uint64_t::Type atomic_fetch_sub(volatile atomic_uint64_t *ptr, - atomic_uint64_t::Type val, - memory_order mo) { - return atomic_fetch_add(ptr, -val, mo); -} - -template <> -inline bool atomic_compare_exchange_strong(volatile atomic_uint64_t *ptr, - atomic_uint64_t::Type *cmp, - atomic_uint64_t::Type xchg, - memory_order mo) { - DCHECK(mo & - (memory_order_relaxed | memory_order_release | memory_order_seq_cst)); - DCHECK(!((uptr)ptr % sizeof(*ptr))); - - typedef atomic_uint64_t::Type Type; - Type cmpv = *cmp; - Type prev; - bool ret = false; - - __spin_lock(&lock.lock); - prev = *(const_cast(&ptr->val_dont_use)); - if (prev == cmpv) { - ret = true; - ptr->val_dont_use = xchg; - } - __spin_unlock(&lock.lock); - - return ret; -} - -template <> -inline atomic_uint64_t::Type atomic_load(const volatile atomic_uint64_t *ptr, - memory_order mo) { - DCHECK(mo & - (memory_order_relaxed | memory_order_release | memory_order_seq_cst)); - DCHECK(!((uptr)ptr % sizeof(*ptr))); - - atomic_uint64_t::Type zero = 0; - volatile atomic_uint64_t *Newptr = - const_cast(ptr); - return atomic_fetch_add(Newptr, zero, mo); -} - -template <> -inline void atomic_store(volatile atomic_uint64_t *ptr, atomic_uint64_t::Type v, - memory_order mo) { - DCHECK(mo & - (memory_order_relaxed | memory_order_release | memory_order_seq_cst)); - DCHECK(!((uptr)ptr % sizeof(*ptr))); - - __spin_lock(&lock.lock); - ptr->val_dont_use = v; - __spin_unlock(&lock.lock); -} - -} // namespace __sanitizer - -#endif // SANITIZER_ATOMIC_CLANG_MIPS_H - diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_other.h b/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_other.h deleted file mode 100644 index 557082a636b8..000000000000 --- a/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_other.h +++ /dev/null @@ -1,85 +0,0 @@ -//===-- sanitizer_atomic_clang_other.h --------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This file is a part of ThreadSanitizer/AddressSanitizer runtime. -// Not intended for direct inclusion. Include sanitizer_atomic.h. -// -//===----------------------------------------------------------------------===// - -#ifndef SANITIZER_ATOMIC_CLANG_OTHER_H -#define SANITIZER_ATOMIC_CLANG_OTHER_H - -namespace __sanitizer { - - -inline void proc_yield(int cnt) { - __asm__ __volatile__("" ::: "memory"); -} - -template -inline typename T::Type atomic_load( - const volatile T *a, memory_order mo) { - DCHECK(mo & (memory_order_relaxed | memory_order_consume - | memory_order_acquire | memory_order_seq_cst)); - DCHECK(!((uptr)a % sizeof(*a))); - typename T::Type v; - - if (sizeof(*a) < 8 || sizeof(void*) == 8) { - // Assume that aligned loads are atomic. - if (mo == memory_order_relaxed) { - v = a->val_dont_use; - } else if (mo == memory_order_consume) { - // Assume that processor respects data dependencies - // (and that compiler won't break them). - __asm__ __volatile__("" ::: "memory"); - v = a->val_dont_use; - __asm__ __volatile__("" ::: "memory"); - } else if (mo == memory_order_acquire) { - __asm__ __volatile__("" ::: "memory"); - v = a->val_dont_use; - __sync_synchronize(); - } else { // seq_cst - // E.g. on POWER we need a hw fence even before the store. - __sync_synchronize(); - v = a->val_dont_use; - __sync_synchronize(); - } - } else { - __atomic_load(const_cast(&a->val_dont_use), &v, - __ATOMIC_SEQ_CST); - } - return v; -} - -template -inline void atomic_store(volatile T *a, typename T::Type v, memory_order mo) { - DCHECK(mo & (memory_order_relaxed | memory_order_release - | memory_order_seq_cst)); - DCHECK(!((uptr)a % sizeof(*a))); - - if (sizeof(*a) < 8 || sizeof(void*) == 8) { - // Assume that aligned stores are atomic. - if (mo == memory_order_relaxed) { - a->val_dont_use = v; - } else if (mo == memory_order_release) { - __sync_synchronize(); - a->val_dont_use = v; - __asm__ __volatile__("" ::: "memory"); - } else { // seq_cst - __sync_synchronize(); - a->val_dont_use = v; - __sync_synchronize(); - } - } else { - __atomic_store(&a->val_dont_use, &v, __ATOMIC_SEQ_CST); - } -} - -} // namespace __sanitizer - -#endif // #ifndef SANITIZER_ATOMIC_CLANG_OTHER_H diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_x86.h b/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_x86.h deleted file mode 100644 index b81a354d2098..000000000000 --- a/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_x86.h +++ /dev/null @@ -1,113 +0,0 @@ -//===-- sanitizer_atomic_clang_x86.h ----------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This file is a part of ThreadSanitizer/AddressSanitizer runtime. -// Not intended for direct inclusion. Include sanitizer_atomic.h. -// -//===----------------------------------------------------------------------===// - -#ifndef SANITIZER_ATOMIC_CLANG_X86_H -#define SANITIZER_ATOMIC_CLANG_X86_H - -namespace __sanitizer { - -inline void proc_yield(int cnt) { - __asm__ __volatile__("" ::: "memory"); - for (int i = 0; i < cnt; i++) - __asm__ __volatile__("pause"); - __asm__ __volatile__("" ::: "memory"); -} - -template -inline typename T::Type atomic_load( - const volatile T *a, memory_order mo) { - DCHECK(mo & (memory_order_relaxed | memory_order_consume - | memory_order_acquire | memory_order_seq_cst)); - DCHECK(!((uptr)a % sizeof(*a))); - typename T::Type v; - - if (sizeof(*a) < 8 || sizeof(void*) == 8) { - // Assume that aligned loads are atomic. - if (mo == memory_order_relaxed) { - v = a->val_dont_use; - } else if (mo == memory_order_consume) { - // Assume that processor respects data dependencies - // (and that compiler won't break them). - __asm__ __volatile__("" ::: "memory"); - v = a->val_dont_use; - __asm__ __volatile__("" ::: "memory"); - } else if (mo == memory_order_acquire) { - __asm__ __volatile__("" ::: "memory"); - v = a->val_dont_use; - // On x86 loads are implicitly acquire. - __asm__ __volatile__("" ::: "memory"); - } else { // seq_cst - // On x86 plain MOV is enough for seq_cst store. - __asm__ __volatile__("" ::: "memory"); - v = a->val_dont_use; - __asm__ __volatile__("" ::: "memory"); - } - } else { - // 64-bit load on 32-bit platform. - __asm__ __volatile__( - "movq %1, %%mm0;" // Use mmx reg for 64-bit atomic moves - "movq %%mm0, %0;" // (ptr could be read-only) - "emms;" // Empty mmx state/Reset FP regs - : "=m" (v) - : "m" (a->val_dont_use) - : // mark the mmx registers as clobbered -#ifdef __MMX__ - "mm0", "mm1", "mm2", "mm3", "mm4", "mm5", "mm6", "mm7", -#endif // #ifdef __MMX__ - "memory"); - } - return v; -} - -template -inline void atomic_store(volatile T *a, typename T::Type v, memory_order mo) { - DCHECK(mo & (memory_order_relaxed | memory_order_release - | memory_order_seq_cst)); - DCHECK(!((uptr)a % sizeof(*a))); - - if (sizeof(*a) < 8 || sizeof(void*) == 8) { - // Assume that aligned stores are atomic. - if (mo == memory_order_relaxed) { - a->val_dont_use = v; - } else if (mo == memory_order_release) { - // On x86 stores are implicitly release. - __asm__ __volatile__("" ::: "memory"); - a->val_dont_use = v; - __asm__ __volatile__("" ::: "memory"); - } else { // seq_cst - // On x86 stores are implicitly release. - __asm__ __volatile__("" ::: "memory"); - a->val_dont_use = v; - __sync_synchronize(); - } - } else { - // 64-bit store on 32-bit platform. - __asm__ __volatile__( - "movq %1, %%mm0;" // Use mmx reg for 64-bit atomic moves - "movq %%mm0, %0;" - "emms;" // Empty mmx state/Reset FP regs - : "=m" (a->val_dont_use) - : "m" (v) - : // mark the mmx registers as clobbered -#ifdef __MMX__ - "mm0", "mm1", "mm2", "mm3", "mm4", "mm5", "mm6", "mm7", -#endif // #ifdef __MMX__ - "memory"); - if (mo == memory_order_seq_cst) - __sync_synchronize(); - } -} - -} // namespace __sanitizer - -#endif // #ifndef SANITIZER_ATOMIC_CLANG_X86_H diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_atomic_msvc.h b/compiler-rt/lib/sanitizer_common/sanitizer_atomic_msvc.h index 31317adcdfc9..d80bfdbf6a08 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_atomic_msvc.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_atomic_msvc.h @@ -70,8 +70,8 @@ inline void proc_yield(int cnt) { template inline typename T::Type atomic_load( const volatile T *a, memory_order mo) { - DCHECK(mo & (memory_order_relaxed | memory_order_consume - | memory_order_acquire | memory_order_seq_cst)); + DCHECK(mo == memory_order_relaxed || mo == memory_order_consume || + mo == memory_order_acquire || mo == memory_order_seq_cst); DCHECK(!((uptr)a % sizeof(*a))); typename T::Type v; // FIXME(dvyukov): 64-bit load is not atomic on 32-bits. @@ -87,8 +87,8 @@ inline typename T::Type atomic_load( template inline void atomic_store(volatile T *a, typename T::Type v, memory_order mo) { - DCHECK(mo & (memory_order_relaxed | memory_order_release - | memory_order_seq_cst)); + DCHECK(mo == memory_order_relaxed || mo == memory_order_release || + mo == memory_order_seq_cst); DCHECK(!((uptr)a % sizeof(*a))); // FIXME(dvyukov): 64-bit store is not atomic on 32-bits. if (mo == memory_order_relaxed) { diff --git a/llvm/utils/gn/secondary/compiler-rt/lib/sanitizer_common/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/lib/sanitizer_common/BUILD.gn index 051907323943..f7f1fce10bf5 100644 --- a/llvm/utils/gn/secondary/compiler-rt/lib/sanitizer_common/BUILD.gn +++ b/llvm/utils/gn/secondary/compiler-rt/lib/sanitizer_common/BUILD.gn @@ -33,9 +33,6 @@ source_set("sources") { "sanitizer_asm.h", "sanitizer_atomic.h", "sanitizer_atomic_clang.h", - "sanitizer_atomic_clang_mips.h", - "sanitizer_atomic_clang_other.h", - "sanitizer_atomic_clang_x86.h", "sanitizer_atomic_msvc.h", "sanitizer_bitvector.h", "sanitizer_bvgraph.h", -- GitLab From a88ea8fbb3953c2fe2887438baf342e381a79d8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?= =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?= =?UTF-8?q?=E3=83=B3=29?= Date: Wed, 17 Apr 2024 08:43:11 -0700 Subject: [PATCH 069/862] [flang][cuda] Update memory effect on fir.cuda_allocate op (#88930) Add MemRead effect on the box operand as the descriptor might be read when performing the allocation of the data. Also update the expected type of the box operand to be a reference. Check in the verifier that this is a reference to a box or class type. This addresses the comment made post commit on #88586 --- flang/include/flang/Optimizer/Dialect/FIROps.td | 2 +- flang/lib/Optimizer/Dialect/FIROps.cpp | 2 +- flang/test/Fir/cuf-invalid.fir | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td index 580e840587ab..92790a691e47 100644 --- a/flang/include/flang/Optimizer/Dialect/FIROps.td +++ b/flang/include/flang/Optimizer/Dialect/FIROps.td @@ -3200,7 +3200,7 @@ def fir_CUDAAllocateOp : fir_Op<"cuda_allocate", [AttrSizedOperandSegments, is initialized before with the standard flang runtime calls. }]; - let arguments = (ins Arg:$box, + let arguments = (ins Arg:$box, Arg, "", [MemWrite]>:$errmsg, Optional:$stream, Arg, "", [MemWrite]>:$pinned, diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp index be27256d911b..5c24c95db427 100644 --- a/flang/lib/Optimizer/Dialect/FIROps.cpp +++ b/flang/lib/Optimizer/Dialect/FIROps.cpp @@ -3998,7 +3998,7 @@ mlir::LogicalResult fir::CUDAAllocateOp::verify() { return emitOpError("pinned and stream cannot appears at the same time"); if (!fir::unwrapRefType(getBox().getType()).isa()) return emitOpError( - "expect box to be a reference to/or a class or box type value"); + "expect box to be a reference to a class or box type value"); if (getSource() && !fir::unwrapRefType(getSource().getType()).isa()) return emitOpError( diff --git a/flang/test/Fir/cuf-invalid.fir b/flang/test/Fir/cuf-invalid.fir index 5d3aa55cf346..6c533a32ccf9 100644 --- a/flang/test/Fir/cuf-invalid.fir +++ b/flang/test/Fir/cuf-invalid.fir @@ -16,7 +16,7 @@ func.func @_QPsub1() { func.func @_QPsub1() { %1 = fir.alloca i32 - // expected-error@+1{{'fir.cuda_allocate' op expect box to be a reference to/or a class or box type value}} + // expected-error@+1{{'fir.cuda_allocate' op expect box to be a reference to a class or box type value}} %2 = fir.cuda_allocate %1 : !fir.ref {cuda_attr = #fir.cuda} -> i32 return } -- GitLab From da70f2cdcde8cb96e75ce0236db1fb5353407a69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?= =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?= =?UTF-8?q?=E3=83=B3=29?= Date: Wed, 17 Apr 2024 08:43:25 -0700 Subject: [PATCH 070/862] [flang][cuda] Lower ALLOCATE for device variable (#88980) Replace the runtime call to `AllocatableAllocate` for CUDA device variable to the newly added `fir.cuda_allocate` operation. --- flang/lib/Lower/Allocatable.cpp | 57 +++++++++-- flang/test/Lower/CUDA/cuda-allocatable.cuf | 107 +++++++++++++++++++++ 2 files changed, 154 insertions(+), 10 deletions(-) create mode 100644 flang/test/Lower/CUDA/cuda-allocatable.cuf diff --git a/flang/lib/Lower/Allocatable.cpp b/flang/lib/Lower/Allocatable.cpp index 42e78fc96e44..1d434d512d0c 100644 --- a/flang/lib/Lower/Allocatable.cpp +++ b/flang/lib/Lower/Allocatable.cpp @@ -14,6 +14,7 @@ #include "flang/Evaluate/tools.h" #include "flang/Lower/AbstractConverter.h" #include "flang/Lower/ConvertType.h" +#include "flang/Lower/ConvertVariable.h" #include "flang/Lower/IterationSpace.h" #include "flang/Lower/Mangler.h" #include "flang/Lower/OpenACC.h" @@ -368,20 +369,17 @@ private: [&](const Fortran::parser::AllocOpt::Mold &mold) { moldExpr = Fortran::semantics::GetExpr(mold.v.value()); }, - [&](const Fortran::parser::AllocOpt::Stream &) { - TODO(loc, "CUDA ALLOCATE(STREAM=)"); + [&](const Fortran::parser::AllocOpt::Stream &stream) { + streamExpr = Fortran::semantics::GetExpr(stream.v.value()); }, - [&](const Fortran::parser::AllocOpt::Pinned &) { - TODO(loc, "CUDA ALLOCATE(PINNED=)"); + [&](const Fortran::parser::AllocOpt::Pinned &pinned) { + pinnedExpr = Fortran::semantics::GetExpr(pinned.v.value()); }, }, allocOption.u); } void lowerAllocation(const Allocation &alloc) { - if (Fortran::semantics::HasCUDAAttr(alloc.getSymbol())) - TODO(loc, "Allocation of variable with CUDA attributes"); - fir::MutableBoxValue boxAddr = genMutableBoxValue(converter, loc, alloc.getAllocObj()); @@ -456,7 +454,8 @@ private: const fir::MutableBoxValue &box) { if (!box.isDerived() && !errorManager.hasStatSpec() && !alloc.type.IsPolymorphic() && !alloc.hasCoarraySpec() && - !useAllocateRuntime && !box.isPointer()) { + !useAllocateRuntime && !box.isPointer() && + !Fortran::semantics::HasCUDAAttr(alloc.getSymbol())) { // Pointers must use PointerAllocate so that their deallocations // can be validated. genInlinedAllocation(alloc, box); @@ -472,7 +471,12 @@ private: genSetType(alloc, box, loc); genSetDeferredLengthParameters(alloc, box); genAllocateObjectBounds(alloc, box); - mlir::Value stat = genRuntimeAllocate(builder, loc, box, errorManager); + mlir::Value stat; + if (!Fortran::semantics::HasCUDAAttr(alloc.getSymbol())) + stat = genRuntimeAllocate(builder, loc, box, errorManager); + else + stat = + genCudaAllocate(builder, loc, box, errorManager, alloc.getSymbol()); fir::factory::syncMutableBoxFromIRBox(builder, loc, box); postAllocationAction(alloc); errorManager.assignStat(builder, loc, stat); @@ -602,7 +606,10 @@ private: genSetDeferredLengthParameters(alloc, box); genAllocateObjectBounds(alloc, box); mlir::Value stat; - if (isSource) + if (Fortran::semantics::HasCUDAAttr(alloc.getSymbol())) + stat = + genCudaAllocate(builder, loc, box, errorManager, alloc.getSymbol()); + else if (isSource) stat = genRuntimeAllocateSource(builder, loc, box, exv, errorManager); else stat = genRuntimeAllocate(builder, loc, box, errorManager); @@ -717,6 +724,34 @@ private: return nullptr; } + mlir::Value genCudaAllocate(fir::FirOpBuilder &builder, mlir::Location loc, + const fir::MutableBoxValue &box, + ErrorManager &errorManager, + const Fortran::semantics::Symbol &sym) { + Fortran::lower::StatementContext stmtCtx; + fir::CUDADataAttributeAttr cudaAttr = + Fortran::lower::translateSymbolCUDADataAttribute(builder.getContext(), + sym); + mlir::Value errmsg = errMsgExpr ? errorManager.errMsgAddr : nullptr; + mlir::Value stream = + streamExpr + ? fir::getBase(converter.genExprValue(loc, *streamExpr, stmtCtx)) + : nullptr; + mlir::Value pinned = + pinnedExpr + ? fir::getBase(converter.genExprAddr(loc, *pinnedExpr, stmtCtx)) + : nullptr; + mlir::Value source = sourceExpr ? fir::getBase(sourceExv) : nullptr; + + // Keep return type the same as a standard AllocatableAllocate call. + mlir::Type retTy = fir::runtime::getModel()(builder.getContext()); + return builder + .create( + loc, retTy, box.getAddr(), errmsg, stream, pinned, source, cudaAttr, + errorManager.hasStatSpec() ? builder.getUnitAttr() : nullptr) + .getResult(); + } + Fortran::lower::AbstractConverter &converter; fir::FirOpBuilder &builder; const Fortran::parser::AllocateStmt &stmt; @@ -724,6 +759,8 @@ private: const Fortran::lower::SomeExpr *moldExpr{nullptr}; const Fortran::lower::SomeExpr *statExpr{nullptr}; const Fortran::lower::SomeExpr *errMsgExpr{nullptr}; + const Fortran::lower::SomeExpr *pinnedExpr{nullptr}; + const Fortran::lower::SomeExpr *streamExpr{nullptr}; // If the allocate has a type spec, lenParams contains the // value of the length parameters that were specified inside. llvm::SmallVector lenParams; diff --git a/flang/test/Lower/CUDA/cuda-allocatable.cuf b/flang/test/Lower/CUDA/cuda-allocatable.cuf new file mode 100644 index 000000000000..55223011e8d9 --- /dev/null +++ b/flang/test/Lower/CUDA/cuda-allocatable.cuf @@ -0,0 +1,107 @@ +! RUN: bbc -emit-hlfir -fcuda %s -o - | FileCheck %s + +! Test lowering of CUDA allocatable allocate/deallocate statements. + +subroutine sub1() + real, allocatable, device :: a(:) + allocate(a(10)) +end subroutine + +! CHECK-LABEL: func.func @_QPsub1() +! CHECK: %[[BOX:.*]] = fir.alloca !fir.box>> {bindc_name = "a", uniq_name = "_QFsub1Ea"} +! CHECK: %[[BOX_DECL:.*]]:2 = hlfir.declare %[[BOX]] {cuda_attr = #fir.cuda, fortran_attrs = #fir.var_attrs, uniq_name = "_QFsub1Ea"} : (!fir.ref>>>) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: fir.call @_FortranAAllocatableSetBounds +! CHECK: %{{.*}} = fir.cuda_allocate %[[BOX_DECL]]#1 : !fir.ref>>> {cuda_attr = #fir.cuda} -> i32 + +subroutine sub2() + real, allocatable, managed :: a(:) + integer :: istat + allocate(a(10), stat=istat) +end subroutine + +! CHECK-LABEL: func.func @_QPsub2() +! CHECK: %[[BOX:.*]] = fir.alloca !fir.box>> {bindc_name = "a", uniq_name = "_QFsub2Ea"} +! CHECK: %[[BOX_DECL:.*]]:2 = hlfir.declare %[[BOX]] {cuda_attr = #fir.cuda, fortran_attrs = #fir.var_attrs, uniq_name = "_QFsub2Ea"} : (!fir.ref>>>) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[ISTAT:.*]] = fir.alloca i32 {bindc_name = "istat", uniq_name = "_QFsub2Eistat"} +! CHECK: %[[ISTAT_DECL:.*]]:2 = hlfir.declare %[[ISTAT]] {uniq_name = "_QFsub2Eistat"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: fir.call @_FortranAAllocatableSetBounds +! CHECK: %[[STAT:.*]] = fir.cuda_allocate %[[BOX_DECL]]#1 : !fir.ref>>> {cuda_attr = #fir.cuda, hasStat} -> i32 +! CHECK: fir.store %[[STAT]] to %[[ISTAT_DECL]]#1 : !fir.ref + +subroutine sub3() + integer, allocatable, pinned :: a(:,:) + logical :: plog + allocate(a(20,30), pinned = plog) +end subroutine + +! CHECK-LABEL: func.func @_QPsub3() +! CHECK: %[[BOX:.*]] = fir.alloca !fir.box>> {bindc_name = "a", uniq_name = "_QFsub3Ea"} +! CHECK: %[[BOX_DECL:.*]]:2 = hlfir.declare %[[BOX]] {cuda_attr = #fir.cuda, fortran_attrs = #fir.var_attrs, uniq_name = "_QFsub3Ea"} : (!fir.ref>>>) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[PLOG:.*]] = fir.alloca !fir.logical<4> {bindc_name = "plog", uniq_name = "_QFsub3Eplog"} +! CHECK: %[[PLOG_DECL:.*]]:2 = hlfir.declare %5 {uniq_name = "_QFsub3Eplog"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK-2: fir.call @_FortranAAllocatableSetBounds +! CHECK: %{{.*}} = fir.cuda_allocate %[[BOX_DECL]]#1 : !fir.ref>>> pinned(%[[PLOG_DECL]]#1 : !fir.ref>) {cuda_attr = #fir.cuda} -> i32 + +subroutine sub4() + real, allocatable, unified :: a(:) + integer :: istream + allocate(a(10), stream=istream) +end subroutine + +! CHECK-LABEL: func.func @_QPsub4() +! CHECK: %[[BOX:.*]] = fir.alloca !fir.box>> {bindc_name = "a", uniq_name = "_QFsub4Ea"} +! CHECK: %[[BOX_DECL:.*]]:2 = hlfir.declare %0 {cuda_attr = #fir.cuda, fortran_attrs = #fir.var_attrs, uniq_name = "_QFsub4Ea"} : (!fir.ref>>>) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[ISTREAM:.*]] = fir.alloca i32 {bindc_name = "istream", uniq_name = "_QFsub4Eistream"} +! CHECK: %[[ISTREAM_DECL:.*]]:2 = hlfir.declare %[[ISTREAM]] {uniq_name = "_QFsub4Eistream"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: fir.call @_FortranAAllocatableSetBounds +! CHECK: %[[STREAM:.*]] = fir.load %[[ISTREAM_DECL]]#0 : !fir.ref +! CHECK: %{{.*}} = fir.cuda_allocate %[[BOX_DECL]]#1 : !fir.ref>>> stream(%[[STREAM]] : i32) {cuda_attr = #fir.cuda} -> i32 + +subroutine sub5() + real, allocatable, device :: a(:) + real, allocatable :: b(:) + allocate(a, source=b) +end subroutine + +! CHECK-LABEL: func.func @_QPsub5() +! CHECK: %[[BOX_A:.*]] = fir.alloca !fir.box>> {bindc_name = "a", uniq_name = "_QFsub5Ea"} +! CHECK: %[[BOX_A_DECL:.*]]:2 = hlfir.declare %[[BOX]] {cuda_attr = #fir.cuda, fortran_attrs = #fir.var_attrs, uniq_name = "_QFsub5Ea"} : (!fir.ref>>>) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[BOX_B:.*]] = fir.alloca !fir.box>> {bindc_name = "b", uniq_name = "_QFsub5Eb"} +! CHECK: %[[BOX_B_DECL:.*]]:2 = hlfir.declare %[[BOX_B]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFsub5Eb"} : (!fir.ref>>>) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[LOAD_B:.*]] = fir.load %[[BOX_B_DECL]]#1 : !fir.ref>>> +! CHECK: fir.call @_FortranAAllocatableSetBounds +! CHECK: %{{.*}} = fir.cuda_allocate %[[BOX_A_DECL]]#1 : !fir.ref>>> source(%[[LOAD_B]] : !fir.box>>) {cuda_attr = #fir.cuda} -> i32 + +subroutine sub6() + real, allocatable, device :: a(:) + real, allocatable :: b(:) + allocate(a, mold=b) +end subroutine + +! CHECK-LABEL: func.func @_QPsub6() +! CHECK: %[[BOX_A:.*]] = fir.alloca !fir.box>> {bindc_name = "a", uniq_name = "_QFsub6Ea"} +! CHECK: %[[BOX_A_DECL:.*]]:2 = hlfir.declare %[[BOX]] {cuda_attr = #fir.cuda, fortran_attrs = #fir.var_attrs, uniq_name = "_QFsub6Ea"} : (!fir.ref>>>) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[BOX_B:.*]] = fir.alloca !fir.box>> {bindc_name = "b", uniq_name = "_QFsub6Eb"} +! CHECK: %[[BOX_B_DECL:.*]]:2 = hlfir.declare %[[BOX_B]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFsub6Eb"} : (!fir.ref>>>) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[LOAD_B:.*]] = fir.load %[[BOX_B_DECL]]#1 : !fir.ref>>> +! CHECK: fir.call @_FortranAAllocatableApplyMold +! CHECK: %{{.*}} = fir.cuda_allocate %[[BOX_A_DECL]]#1 : !fir.ref>>> {cuda_attr = #fir.cuda} -> i32 + +subroutine sub7() + real, allocatable, device :: a(:) + integer :: istat + character(50) :: err + allocate(a(100), stat=istat, errmsg=err) +end subroutine + +! CHECK-LABEL: func.func @_QPsub7() +! CHECK: %[[BOX:.*]] = fir.alloca !fir.box>> {bindc_name = "a", uniq_name = "_QFsub7Ea"} +! CHECK: %[[BOX_DECL:.*]]:2 = hlfir.declare %[[BOX]] {cuda_attr = #fir.cuda, fortran_attrs = #fir.var_attrs, uniq_name = "_QFsub7Ea"} : (!fir.ref>>>) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[ERR:.*]] = fir.alloca !fir.char<1,50> {bindc_name = "err", uniq_name = "_QFsub7Eerr"} +! CHECK: %[[ERR_DECL:.*]]:2 = hlfir.declare %[[ERR]] typeparams %{{.*}} {uniq_name = "_QFsub7Eerr"} : (!fir.ref>, index) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[ISTAT:.*]] = fir.alloca i32 {bindc_name = "istat", uniq_name = "_QFsub7Eistat"} +! CHECK: %[[ISTAT_DECL:.*]]:2 = hlfir.declare %[[ISTAT]] {uniq_name = "_QFsub7Eistat"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[ERR_BOX:.*]] = fir.embox %[[ERR_DECL]]#1 : (!fir.ref>) -> !fir.box> +! CHECK: fir.call @_FortranAAllocatableSetBounds +! CHECK: %[[STAT:.*]] = fir.cuda_allocate %[[BOX_DECL]]#1 : !fir.ref>>> errmsg(%[[ERR_BOX]] : !fir.box>) {cuda_attr = #fir.cuda, hasStat} -> i32 +! CHECK: fir.store %[[STAT]] to %[[ISTAT_DECL]]#1 : !fir.ref -- GitLab From 19c6a7feca6e1558ef7cbe18efd2477c1126899d Mon Sep 17 00:00:00 2001 From: Alexandros Lamprineas Date: Wed, 17 Apr 2024 17:16:58 +0100 Subject: [PATCH 071/862] [FMV] Remove useless features according the latest ACLE spec. (#88965) As explained in https://github.com/ARM-software/acle/pull/315 we are deprecating features which aren't adding any value. These are: sha1, pmull, dit, dgh, ebf16, sve-bf16, sve-ebf16, sve-i8mm, sve2-pmull128, memtag2, memtag3, ssbs2, bti, ls64_v, ls64_accdata --- clang/test/CodeGen/aarch64-cpu-supports.c | 24 +- .../CodeGen/aarch64-mixed-target-attributes.c | 12 +- .../test/CodeGen/attr-target-clones-aarch64.c | 77 ++--- clang/test/CodeGen/attr-target-version.c | 326 +++++++++--------- .../CodeGenCXX/attr-target-clones-aarch64.cpp | 41 +-- clang/test/CodeGenCXX/attr-target-version.cpp | 22 +- clang/test/Sema/aarch64-cpu-supports.c | 2 +- clang/test/Sema/attr-target-clones-aarch64.c | 21 +- clang/test/Sema/attr-target-version.c | 18 +- clang/test/SemaCXX/attr-target-version.cpp | 16 +- compiler-rt/lib/builtins/cpu_model/aarch64.c | 17 +- .../builtins/cpu_model/aarch64/fmv/apple.inc | 6 +- .../cpu_model/aarch64/fmv/fuchsia.inc | 4 - .../builtins/cpu_model/aarch64/fmv/mrs.inc | 42 +-- .../llvm/TargetParser/AArch64TargetParser.h | 36 +- 15 files changed, 287 insertions(+), 377 deletions(-) diff --git a/clang/test/CodeGen/aarch64-cpu-supports.c b/clang/test/CodeGen/aarch64-cpu-supports.c index c54b7475a3fd..7fad9724dfb6 100644 --- a/clang/test/CodeGen/aarch64-cpu-supports.c +++ b/clang/test/CodeGen/aarch64-cpu-supports.c @@ -1,15 +1,17 @@ // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --check-globals --version 2 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -emit-llvm -o - %s | FileCheck %s +//. // CHECK: @__aarch64_cpu_features = external dso_local global { i64 } +//. // CHECK-LABEL: define dso_local i32 @main // CHECK-SAME: () #[[ATTR0:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[RETVAL:%.*]] = alloca i32, align 4 // CHECK-NEXT: store i32 0, ptr [[RETVAL]], align 4 // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 70368744177664 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 70368744177664 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 34359738368 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 34359738368 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK: if.then: @@ -17,8 +19,8 @@ // CHECK-NEXT: br label [[RETURN:%.*]] // CHECK: if.end: // CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 9070970929152 -// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 9070970929152 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 17716740096 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 17716740096 // CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] // CHECK-NEXT: br i1 [[TMP7]], label [[IF_THEN1:%.*]], label [[IF_END2:%.*]] // CHECK: if.then1: @@ -26,8 +28,8 @@ // CHECK-NEXT: br label [[RETURN]] // CHECK: if.end2: // CHECK-NEXT: [[TMP8:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP9:%.*]] = and i64 [[TMP8]], 166633186212708352 -// CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 166633186212708352 +// CHECK-NEXT: [[TMP9:%.*]] = and i64 [[TMP8]], 5222680231936 +// CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 5222680231936 // CHECK-NEXT: [[TMP11:%.*]] = and i1 true, [[TMP10]] // CHECK-NEXT: br i1 [[TMP11]], label [[IF_THEN3:%.*]], label [[IF_END4:%.*]] // CHECK: if.then3: @@ -49,10 +51,10 @@ int main(void) { if (__builtin_cpu_supports("sb")) return 1; - if (__builtin_cpu_supports("sve2-pmull128+memtag")) + if (__builtin_cpu_supports("sve2-aes+memtag")) return 2; - if (__builtin_cpu_supports("sme2+ls64_v+wfxt")) + if (__builtin_cpu_supports("sme2+ls64+wfxt")) return 3; if (__builtin_cpu_supports("avx2")) @@ -60,3 +62,9 @@ int main(void) { return 0; } +//. +// CHECK: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +//. +// CHECK: [[META0:![0-9]+]] = !{i32 1, !"wchar_size", i32 4} +// CHECK: [[META1:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"} +//. diff --git a/clang/test/CodeGen/aarch64-mixed-target-attributes.c b/clang/test/CodeGen/aarch64-mixed-target-attributes.c index aef6ce36ab1c..be290ff9ecee 100644 --- a/clang/test/CodeGen/aarch64-mixed-target-attributes.c +++ b/clang/test/CodeGen/aarch64-mixed-target-attributes.c @@ -69,8 +69,8 @@ __attribute__((target_version("jscvt"))) int default_def_with_version_decls(void // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1048576 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1048576 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 131072 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 131072 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: @@ -143,8 +143,8 @@ __attribute__((target_version("jscvt"))) int default_def_with_version_decls(void // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1048576 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1048576 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 131072 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 131072 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: @@ -210,8 +210,8 @@ __attribute__((target_version("jscvt"))) int default_def_with_version_decls(void // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1048576 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1048576 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 131072 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 131072 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: diff --git a/clang/test/CodeGen/attr-target-clones-aarch64.c b/clang/test/CodeGen/attr-target-clones-aarch64.c index 8c8b951e9118..c715001f6a72 100644 --- a/clang/test/CodeGen/attr-target-clones-aarch64.c +++ b/clang/test/CodeGen/attr-target-clones-aarch64.c @@ -3,7 +3,7 @@ // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature -fmv -S -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK-NOFMV int __attribute__((target_clones("lse+aes", "sve2"))) ftc(void) { return 0; } -int __attribute__((target_clones("sha2", "sha2+memtag2", " default "))) ftc_def(void) { return 1; } +int __attribute__((target_clones("sha2", "sha2+memtag", " default "))) ftc_def(void) { return 1; } int __attribute__((target_clones("sha2", "default"))) ftc_dup1(void) { return 2; } int __attribute__((target_clones("fp", "crc+dotprod"))) ftc_dup2(void) { return 3; } int foo() { @@ -12,7 +12,7 @@ int foo() { inline int __attribute__((target_clones("rng+simd", "rcpc+predres", "sve2-aes+wfxt"))) ftc_inline1(void) { return 1; } inline int __attribute__((target_clones("fp16", "fcma+sve2-bitperm", "default"))) ftc_inline2(void); -inline int __attribute__((target_clones("bti", "sve+sb"))) ftc_inline3(void) { return 3; } +inline int __attribute__((target_clones("mops", "sve+sb"))) ftc_inline3(void) { return 3; } int __attribute__((target_clones("default"))) ftc_direct(void) { return 4; } @@ -56,16 +56,16 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default")) // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 16512 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 16512 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 8320 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 8320 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: // CHECK-NEXT: ret ptr @ftc._MaesMlse // CHECK: resolver_else: // CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 68719476736 -// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 68719476736 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 268435456 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 268435456 // CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] // CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] // CHECK: resolver_return1: @@ -81,7 +81,7 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default")) // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: @ftc_def._Mmemtag2Msha2( +// CHECK-LABEL: @ftc_def._MmemtagMsha2( // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 1 // @@ -90,16 +90,16 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default")) // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 17592186048512 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 17592186048512 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 17179871232 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 17179871232 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: -// CHECK-NEXT: ret ptr @ftc_def._Mmemtag2Msha2 +// CHECK-NEXT: ret ptr @ftc_def._MmemtagMsha2 // CHECK: resolver_else: // CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 4096 -// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 4096 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 2048 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 2048 // CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] // CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] // CHECK: resolver_return1: @@ -118,8 +118,8 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default")) // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 4096 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 4096 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 2048 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 2048 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: @@ -198,16 +198,16 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default")) // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 18014535948435456 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 18014535948435456 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 550292684800 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 550292684800 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: // CHECK-NEXT: ret ptr @ftc_inline1._Msve2-aesMwfxt // CHECK: resolver_else: // CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 140737492549632 -// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 140737492549632 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 68720001024 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 68720001024 // CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] // CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] // CHECK: resolver_return1: @@ -228,16 +228,16 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default")) // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 549757911040 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 549757911040 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1074003968 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1074003968 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: // CHECK-NEXT: ret ptr @ftc_inline2._MfcmaMsve2-bitperm // CHECK: resolver_else: // CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 65536 -// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 65536 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 16384 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 16384 // CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] // CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] // CHECK: resolver_return1: @@ -250,20 +250,20 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default")) // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 70369817919488 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 70369817919488 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 34393292800 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 34393292800 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: // CHECK-NEXT: ret ptr @ftc_inline3._MsbMsve // CHECK: resolver_else: // CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 1125899906842624 -// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 1125899906842624 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 17592186044416 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 17592186044416 // CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] // CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] // CHECK: resolver_return1: -// CHECK-NEXT: ret ptr @ftc_inline3._Mbti +// CHECK-NEXT: ret ptr @ftc_inline3._Mmops // CHECK: resolver_else2: // CHECK-NEXT: ret ptr @ftc_inline3.default // @@ -329,7 +329,7 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default")) // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: @ftc_inline3._Mbti( +// CHECK-LABEL: @ftc_inline3._Mmops( // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 3 // @@ -407,17 +407,16 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default")) // CHECK: attributes #[[ATTR0:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+lse,+neon" } // CHECK: attributes #[[ATTR1:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve,+sve2" } // CHECK: attributes #[[ATTR2:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,+sha2" } -// CHECK: attributes #[[ATTR3:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+mte,+neon,+sha2" } -// CHECK: attributes #[[ATTR4:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon" } -// CHECK: attributes #[[ATTR5:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc,+dotprod,+fp-armv8,+neon" } -// CHECK: attributes #[[ATTR6:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" } -// CHECK: attributes #[[ATTR7:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon" } -// CHECK: attributes #[[ATTR8:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+complxnum,+fp-armv8,+fullfp16,+neon,+sve,+sve2,+sve2-bitperm" } -// CHECK: attributes #[[ATTR9:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,+rand" } -// CHECK: attributes #[[ATTR10:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+predres,+rcpc" } -// CHECK: attributes #[[ATTR11:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve,+sve2,+sve2-aes,+wfxt" } -// CHECK: attributes #[[ATTR12:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bti" } -// CHECK: attributes #[[ATTR13:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sb,+sve" } +// CHECK: attributes #[[ATTR3:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon" } +// CHECK: attributes #[[ATTR4:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc,+dotprod,+fp-armv8,+neon" } +// CHECK: attributes #[[ATTR5:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +// CHECK: attributes #[[ATTR6:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon" } +// CHECK: attributes #[[ATTR7:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+complxnum,+fp-armv8,+fullfp16,+neon,+sve,+sve2,+sve2-bitperm" } +// CHECK: attributes #[[ATTR8:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,+rand" } +// CHECK: attributes #[[ATTR9:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+predres,+rcpc" } +// CHECK: attributes #[[ATTR10:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve,+sve2,+sve2-aes,+wfxt" } +// CHECK: attributes #[[ATTR11:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops" } +// CHECK: attributes #[[ATTR12:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sb,+sve" } //. // CHECK-NOFMV: attributes #[[ATTR0:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fmv" } // CHECK-NOFMV: attributes #[[ATTR1:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fmv" } diff --git a/clang/test/CodeGen/attr-target-version.c b/clang/test/CodeGen/attr-target-version.c index dd4cbbf5a898..e71370e6d91d 100644 --- a/clang/test/CodeGen/attr-target-version.c +++ b/clang/test/CodeGen/attr-target-version.c @@ -5,11 +5,11 @@ int __attribute__((target_version("rng+flagm+fp16fml"))) fmv(void) { return 1; } int __attribute__((target_version("flagm2+sme-i16i64"))) fmv(void) { return 2; } int __attribute__((target_version("lse+sha2"))) fmv(void) { return 3; } -int __attribute__((target_version("dotprod+ls64_accdata"))) fmv(void) { return 4; } +int __attribute__((target_version("dotprod+ls64"))) fmv(void) { return 4; } int __attribute__((target_version("fp16fml+memtag"))) fmv(void) { return 5; } int __attribute__((target_version("fp+aes"))) fmv(void) { return 6; } -int __attribute__((target_version("crc+ls64_v"))) fmv(void) { return 7; } -int __attribute__((target_version("bti"))) fmv(void) { return 8; } +int __attribute__((target_version("crc+ls64"))) fmv(void) { return 7; } +int __attribute__((target_version("mops"))) fmv(void) { return 8; } int __attribute__((target_version("sme2"))) fmv(void) { return 9; } int __attribute__((target_version("default"))) fmv(void); int __attribute__((target_version("ls64+simd"))) fmv_one(void) { return 1; } @@ -17,25 +17,25 @@ int __attribute__((target_version("dpb"))) fmv_one(void) { return 2; } int __attribute__((target_version("default"))) fmv_one(void); int __attribute__((target_version("fp"))) fmv_two(void) { return 1; } int __attribute__((target_version("simd"))) fmv_two(void) { return 2; } -int __attribute__((target_version("dgh"))) fmv_two(void) { return 3; } +int __attribute__((target_version("frintts"))) fmv_two(void) { return 3; } int __attribute__((target_version("fp16+simd"))) fmv_two(void) { return 4; } int __attribute__((target_version("default"))) fmv_two(void); int foo() { return fmv()+fmv_one()+fmv_two(); } -inline int __attribute__((target_version("sha1+pmull+f64mm"))) fmv_inline(void) { return 1; } +inline int __attribute__((target_version("crypto+f64mm"))) fmv_inline(void) { return 1; } inline int __attribute__((target_version("fp16+fcma+rdma+sme+ fp16 "))) fmv_inline(void) { return 2; } inline int __attribute__((target_version("sha3+i8mm+f32mm"))) fmv_inline(void) { return 12; } -inline int __attribute__((target_version("dit+sve-ebf16"))) fmv_inline(void) { return 8; } +inline int __attribute__((target_version("sme2+ssbs"))) fmv_inline(void) { return 8; } inline int __attribute__((target_version("dpb+rcpc2 "))) fmv_inline(void) { return 6; } inline int __attribute__((target_version(" dpb2 + jscvt"))) fmv_inline(void) { return 7; } inline int __attribute__((target_version("rcpc+frintts"))) fmv_inline(void) { return 3; } -inline int __attribute__((target_version("sve+sve-bf16"))) fmv_inline(void) { return 4; } +inline int __attribute__((target_version("sve+sme"))) fmv_inline(void) { return 4; } inline int __attribute__((target_version("sve2-aes+sve2-sha3"))) fmv_inline(void) { return 5; } -inline int __attribute__((target_version("sve2+sve2-pmull128+sve2-bitperm"))) fmv_inline(void) { return 9; } -inline int __attribute__((target_version("sve2-sm4+memtag2"))) fmv_inline(void) { return 10; } -inline int __attribute__((target_version("memtag3+rcpc3+mops"))) fmv_inline(void) { return 11; } +inline int __attribute__((target_version("sve2+sve2-aes+sve2-bitperm"))) fmv_inline(void) { return 9; } +inline int __attribute__((target_version("sve2-sm4+memtag"))) fmv_inline(void) { return 10; } +inline int __attribute__((target_version("memtag+rcpc3+mops"))) fmv_inline(void) { return 11; } inline int __attribute__((target_version("aes+dotprod"))) fmv_inline(void) { return 13; } inline int __attribute__((target_version("simd+fp16fml"))) fmv_inline(void) { return 14; } inline int __attribute__((target_version("fp+sm4"))) fmv_inline(void) { return 15; } @@ -186,7 +186,7 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv._MdotprodMls64_accdata +// CHECK-LABEL: define {{[^@]+}}@fmv._MdotprodMls64 // CHECK-SAME: () #[[ATTR3:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 4 @@ -207,14 +207,14 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv._McrcMls64_v +// CHECK-LABEL: define {{[^@]+}}@fmv._McrcMls64 // CHECK-SAME: () #[[ATTR6:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 7 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv._Mbti +// CHECK-LABEL: define {{[^@]+}}@fmv._Mmops // CHECK-SAME: () #[[ATTR7:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 8 @@ -256,7 +256,7 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_two._Mdgh +// CHECK-LABEL: define {{[^@]+}}@fmv_two._Mfrintts // CHECK-SAME: () #[[ATTR11:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 3 @@ -271,7 +271,7 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@foo -// CHECK-SAME: () #[[ATTR11]] { +// CHECK-SAME: () #[[ATTR13:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[CALL:%.*]] = call i32 @fmv() // CHECK-NEXT: [[CALL1:%.*]] = call i32 @fmv_one() @@ -293,68 +293,68 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // CHECK-NEXT: ret ptr @fmv._MflagmMfp16fmlMrng // CHECK: resolver_else: // CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 72057594037927940 -// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 72057594037927940 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 2199023255556 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 2199023255556 // CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] // CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] // CHECK: resolver_return1: // CHECK-NEXT: ret ptr @fmv._Mflagm2Msme-i16i64 // CHECK: resolver_else2: // CHECK-NEXT: [[TMP8:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP9:%.*]] = and i64 [[TMP8]], 9007199254741008 -// CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 9007199254741008 +// CHECK-NEXT: [[TMP9:%.*]] = and i64 [[TMP8]], 274877906960 +// CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 274877906960 // CHECK-NEXT: [[TMP11:%.*]] = and i1 true, [[TMP10]] // CHECK-NEXT: br i1 [[TMP11]], label [[RESOLVER_RETURN3:%.*]], label [[RESOLVER_ELSE4:%.*]] // CHECK: resolver_return3: -// CHECK-NEXT: ret ptr @fmv._MdotprodMls64_accdata +// CHECK-NEXT: ret ptr @fmv._MdotprodMls64 // CHECK: resolver_else4: // CHECK-NEXT: [[TMP12:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP13:%.*]] = and i64 [[TMP12]], 4503599627371520 -// CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 4503599627371520 +// CHECK-NEXT: [[TMP13:%.*]] = and i64 [[TMP12]], 274877907968 +// CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 274877907968 // CHECK-NEXT: [[TMP15:%.*]] = and i1 true, [[TMP14]] // CHECK-NEXT: br i1 [[TMP15]], label [[RESOLVER_RETURN5:%.*]], label [[RESOLVER_ELSE6:%.*]] // CHECK: resolver_return5: -// CHECK-NEXT: ret ptr @fmv._McrcMls64_v +// CHECK-NEXT: ret ptr @fmv._McrcMls64 // CHECK: resolver_else6: // CHECK-NEXT: [[TMP16:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP17:%.*]] = and i64 [[TMP16]], 8796093022216 -// CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[TMP17]], 8796093022216 +// CHECK-NEXT: [[TMP17:%.*]] = and i64 [[TMP16]], 17179869192 +// CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[TMP17]], 17179869192 // CHECK-NEXT: [[TMP19:%.*]] = and i1 true, [[TMP18]] // CHECK-NEXT: br i1 [[TMP19]], label [[RESOLVER_RETURN7:%.*]], label [[RESOLVER_ELSE8:%.*]] // CHECK: resolver_return7: // CHECK-NEXT: ret ptr @fmv._Mfp16fmlMmemtag // CHECK: resolver_else8: // CHECK-NEXT: [[TMP20:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP21:%.*]] = and i64 [[TMP20]], 16640 -// CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[TMP21]], 16640 +// CHECK-NEXT: [[TMP21:%.*]] = and i64 [[TMP20]], 8448 +// CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[TMP21]], 8448 // CHECK-NEXT: [[TMP23:%.*]] = and i1 true, [[TMP22]] // CHECK-NEXT: br i1 [[TMP23]], label [[RESOLVER_RETURN9:%.*]], label [[RESOLVER_ELSE10:%.*]] // CHECK: resolver_return9: // CHECK-NEXT: ret ptr @fmv._MaesMfp // CHECK: resolver_else10: // CHECK-NEXT: [[TMP24:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP25:%.*]] = and i64 [[TMP24]], 4224 -// CHECK-NEXT: [[TMP26:%.*]] = icmp eq i64 [[TMP25]], 4224 +// CHECK-NEXT: [[TMP25:%.*]] = and i64 [[TMP24]], 2176 +// CHECK-NEXT: [[TMP26:%.*]] = icmp eq i64 [[TMP25]], 2176 // CHECK-NEXT: [[TMP27:%.*]] = and i1 true, [[TMP26]] // CHECK-NEXT: br i1 [[TMP27]], label [[RESOLVER_RETURN11:%.*]], label [[RESOLVER_ELSE12:%.*]] // CHECK: resolver_return11: // CHECK-NEXT: ret ptr @fmv._MlseMsha2 // CHECK: resolver_else12: // CHECK-NEXT: [[TMP28:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP29:%.*]] = and i64 [[TMP28]], 144115188075855872 -// CHECK-NEXT: [[TMP30:%.*]] = icmp eq i64 [[TMP29]], 144115188075855872 +// CHECK-NEXT: [[TMP29:%.*]] = and i64 [[TMP28]], 17592186044416 +// CHECK-NEXT: [[TMP30:%.*]] = icmp eq i64 [[TMP29]], 17592186044416 // CHECK-NEXT: [[TMP31:%.*]] = and i1 true, [[TMP30]] // CHECK-NEXT: br i1 [[TMP31]], label [[RESOLVER_RETURN13:%.*]], label [[RESOLVER_ELSE14:%.*]] // CHECK: resolver_return13: -// CHECK-NEXT: ret ptr @fmv._Msme2 +// CHECK-NEXT: ret ptr @fmv._Mmops // CHECK: resolver_else14: // CHECK-NEXT: [[TMP32:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP33:%.*]] = and i64 [[TMP32]], 1125899906842624 -// CHECK-NEXT: [[TMP34:%.*]] = icmp eq i64 [[TMP33]], 1125899906842624 +// CHECK-NEXT: [[TMP33:%.*]] = and i64 [[TMP32]], 4398046511104 +// CHECK-NEXT: [[TMP34:%.*]] = icmp eq i64 [[TMP33]], 4398046511104 // CHECK-NEXT: [[TMP35:%.*]] = and i1 true, [[TMP34]] // CHECK-NEXT: br i1 [[TMP35]], label [[RESOLVER_RETURN15:%.*]], label [[RESOLVER_ELSE16:%.*]] // CHECK: resolver_return15: -// CHECK-NEXT: ret ptr @fmv._Mbti +// CHECK-NEXT: ret ptr @fmv._Msme2 // CHECK: resolver_else16: // CHECK-NEXT: ret ptr @fmv.default // @@ -363,16 +363,16 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 2251799813685760 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 2251799813685760 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 274877907456 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 274877907456 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: // CHECK-NEXT: ret ptr @fmv_one._Mls64Msimd // CHECK: resolver_else: // CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 262144 -// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 262144 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 32768 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 32768 // CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] // CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] // CHECK: resolver_return1: @@ -385,20 +385,20 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 66048 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 66048 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 16896 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 16896 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: // CHECK-NEXT: ret ptr @fmv_two._Mfp16Msimd // CHECK: resolver_else: // CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 33554432 -// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 33554432 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 2097152 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 2097152 // CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] // CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] // CHECK: resolver_return1: -// CHECK-NEXT: ret ptr @fmv_two._Mdgh +// CHECK-NEXT: ret ptr @fmv_two._Mfrintts // CHECK: resolver_else2: // CHECK-NEXT: [[TMP8:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 // CHECK-NEXT: [[TMP9:%.*]] = and i64 [[TMP8]], 512 @@ -421,49 +421,49 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_e.default -// CHECK-SAME: () #[[ATTR11]] { +// CHECK-SAME: () #[[ATTR13]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 20 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_d._Msb -// CHECK-SAME: () #[[ATTR13:[0-9]+]] { +// CHECK-SAME: () #[[ATTR14:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_d.default -// CHECK-SAME: () #[[ATTR11]] { +// CHECK-SAME: () #[[ATTR13]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_default -// CHECK-SAME: () #[[ATTR11]] { +// CHECK-SAME: () #[[ATTR13]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 111 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_c._Mssbs -// CHECK-SAME: () #[[ATTR11]] { +// CHECK-SAME: () #[[ATTR13]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret void // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_c.default -// CHECK-SAME: () #[[ATTR11]] { +// CHECK-SAME: () #[[ATTR13]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret void // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@goo -// CHECK-SAME: () #[[ATTR11]] { +// CHECK-SAME: () #[[ATTR13]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[CALL:%.*]] = call i32 @fmv_inline() // CHECK-NEXT: [[CALL1:%.*]] = call i32 @fmv_e() @@ -477,96 +477,96 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 4398048673856 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 4398048673856 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 8590213184 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 8590213184 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: // CHECK-NEXT: ret ptr @fmv_inline._MfcmaMfp16MrdmMsme // CHECK: resolver_else: // CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 864726312827224064 -// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 864726312827224064 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 26405458935808 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 26405458935808 // CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] // CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] // CHECK: resolver_return1: -// CHECK-NEXT: ret ptr @fmv_inline._Mmemtag3MmopsMrcpc3 +// CHECK-NEXT: ret ptr @fmv_inline._MmemtagMmopsMrcpc3 // CHECK: resolver_else2: // CHECK-NEXT: [[TMP8:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP9:%.*]] = and i64 [[TMP8]], 893353197568 -// CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 893353197568 +// CHECK-NEXT: [[TMP9:%.*]] = and i64 [[TMP8]], 1879048192 +// CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 1879048192 // CHECK-NEXT: [[TMP11:%.*]] = and i1 true, [[TMP10]] // CHECK-NEXT: br i1 [[TMP11]], label [[RESOLVER_RETURN3:%.*]], label [[RESOLVER_ELSE4:%.*]] // CHECK: resolver_return3: -// CHECK-NEXT: ret ptr @fmv_inline._Msve2Msve2-bitpermMsve2-pmull128 +// CHECK-NEXT: ret ptr @fmv_inline._Msve2Msve2-aesMsve2-bitperm // CHECK: resolver_else4: // CHECK-NEXT: [[TMP12:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP13:%.*]] = and i64 [[TMP12]], 34359773184 -// CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 34359773184 +// CHECK-NEXT: [[TMP13:%.*]] = and i64 [[TMP12]], 71307264 +// CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 71307264 // CHECK-NEXT: [[TMP15:%.*]] = and i1 true, [[TMP14]] // CHECK-NEXT: br i1 [[TMP15]], label [[RESOLVER_RETURN5:%.*]], label [[RESOLVER_ELSE6:%.*]] // CHECK: resolver_return5: -// CHECK-NEXT: ret ptr @fmv_inline._Mf64mmMpmullMsha1 +// CHECK-NEXT: ret ptr @fmv_inline._Mf32mmMi8mmMsha3 // CHECK: resolver_else6: // CHECK-NEXT: [[TMP16:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP17:%.*]] = and i64 [[TMP16]], 17246986240 -// CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[TMP17]], 17246986240 +// CHECK-NEXT: [[TMP17:%.*]] = and i64 [[TMP16]], 4535485464576 +// CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[TMP17]], 4535485464576 // CHECK-NEXT: [[TMP19:%.*]] = and i1 true, [[TMP18]] // CHECK-NEXT: br i1 [[TMP19]], label [[RESOLVER_RETURN7:%.*]], label [[RESOLVER_ELSE8:%.*]] // CHECK: resolver_return7: -// CHECK-NEXT: ret ptr @fmv_inline._Mf32mmMi8mmMsha3 +// CHECK-NEXT: ret ptr @fmv_inline._Msme2Mssbs // CHECK: resolver_else8: // CHECK-NEXT: [[TMP20:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP21:%.*]] = and i64 [[TMP20]], 19791209299968 -// CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[TMP21]], 19791209299968 +// CHECK-NEXT: [[TMP21:%.*]] = and i64 [[TMP20]], 21474836480 +// CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[TMP21]], 21474836480 // CHECK-NEXT: [[TMP23:%.*]] = and i1 true, [[TMP22]] // CHECK-NEXT: br i1 [[TMP23]], label [[RESOLVER_RETURN9:%.*]], label [[RESOLVER_ELSE10:%.*]] // CHECK: resolver_return9: -// CHECK-NEXT: ret ptr @fmv_inline._Mmemtag2Msve2-sm4 +// CHECK-NEXT: ret ptr @fmv_inline._MmemtagMsve2-sm4 // CHECK: resolver_else10: // CHECK-NEXT: [[TMP24:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP25:%.*]] = and i64 [[TMP24]], 1236950581248 -// CHECK-NEXT: [[TMP26:%.*]] = icmp eq i64 [[TMP25]], 1236950581248 +// CHECK-NEXT: [[TMP25:%.*]] = and i64 [[TMP24]], 8623489024 +// CHECK-NEXT: [[TMP26:%.*]] = icmp eq i64 [[TMP25]], 8623489024 // CHECK-NEXT: [[TMP27:%.*]] = and i1 true, [[TMP26]] // CHECK-NEXT: br i1 [[TMP27]], label [[RESOLVER_RETURN11:%.*]], label [[RESOLVER_ELSE12:%.*]] // CHECK: resolver_return11: -// CHECK-NEXT: ret ptr @fmv_inline._Msve2-aesMsve2-sha3 +// CHECK-NEXT: ret ptr @fmv_inline._MsmeMsve // CHECK: resolver_else12: // CHECK-NEXT: [[TMP28:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP29:%.*]] = and i64 [[TMP28]], 4295098368 -// CHECK-NEXT: [[TMP30:%.*]] = icmp eq i64 [[TMP29]], 4295098368 +// CHECK-NEXT: [[TMP29:%.*]] = and i64 [[TMP28]], 2684354560 +// CHECK-NEXT: [[TMP30:%.*]] = icmp eq i64 [[TMP29]], 2684354560 // CHECK-NEXT: [[TMP31:%.*]] = and i1 true, [[TMP30]] // CHECK-NEXT: br i1 [[TMP31]], label [[RESOLVER_RETURN13:%.*]], label [[RESOLVER_ELSE14:%.*]] // CHECK: resolver_return13: -// CHECK-NEXT: ret ptr @fmv_inline._MditMsve-ebf16 +// CHECK-NEXT: ret ptr @fmv_inline._Msve2-aesMsve2-sha3 // CHECK: resolver_else14: // CHECK-NEXT: [[TMP32:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP33:%.*]] = and i64 [[TMP32]], 3221225472 -// CHECK-NEXT: [[TMP34:%.*]] = icmp eq i64 [[TMP33]], 3221225472 +// CHECK-NEXT: [[TMP33:%.*]] = and i64 [[TMP32]], 281475110928384 +// CHECK-NEXT: [[TMP34:%.*]] = icmp eq i64 [[TMP33]], 281475110928384 // CHECK-NEXT: [[TMP35:%.*]] = and i1 true, [[TMP34]] // CHECK-NEXT: br i1 [[TMP35]], label [[RESOLVER_RETURN15:%.*]], label [[RESOLVER_ELSE16:%.*]] // CHECK: resolver_return15: -// CHECK-NEXT: ret ptr @fmv_inline._MsveMsve-bf16 +// CHECK-NEXT: ret ptr @fmv_inline._McryptoMf64mm // CHECK: resolver_else16: // CHECK-NEXT: [[TMP36:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP37:%.*]] = and i64 [[TMP36]], 20971520 -// CHECK-NEXT: [[TMP38:%.*]] = icmp eq i64 [[TMP37]], 20971520 +// CHECK-NEXT: [[TMP37:%.*]] = and i64 [[TMP36]], 2621440 +// CHECK-NEXT: [[TMP38:%.*]] = icmp eq i64 [[TMP37]], 2621440 // CHECK-NEXT: [[TMP39:%.*]] = and i1 true, [[TMP38]] // CHECK-NEXT: br i1 [[TMP39]], label [[RESOLVER_RETURN17:%.*]], label [[RESOLVER_ELSE18:%.*]] // CHECK: resolver_return17: // CHECK-NEXT: ret ptr @fmv_inline._MfrinttsMrcpc // CHECK: resolver_else18: // CHECK-NEXT: [[TMP40:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP41:%.*]] = and i64 [[TMP40]], 8650752 -// CHECK-NEXT: [[TMP42:%.*]] = icmp eq i64 [[TMP41]], 8650752 +// CHECK-NEXT: [[TMP41:%.*]] = and i64 [[TMP40]], 1081344 +// CHECK-NEXT: [[TMP42:%.*]] = icmp eq i64 [[TMP41]], 1081344 // CHECK-NEXT: [[TMP43:%.*]] = and i1 true, [[TMP42]] // CHECK-NEXT: br i1 [[TMP43]], label [[RESOLVER_RETURN19:%.*]], label [[RESOLVER_ELSE20:%.*]] // CHECK: resolver_return19: // CHECK-NEXT: ret ptr @fmv_inline._MdpbMrcpc2 // CHECK: resolver_else20: // CHECK-NEXT: [[TMP44:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP45:%.*]] = and i64 [[TMP44]], 1572864 -// CHECK-NEXT: [[TMP46:%.*]] = icmp eq i64 [[TMP45]], 1572864 +// CHECK-NEXT: [[TMP45:%.*]] = and i64 [[TMP44]], 196608 +// CHECK-NEXT: [[TMP46:%.*]] = icmp eq i64 [[TMP45]], 196608 // CHECK-NEXT: [[TMP47:%.*]] = and i1 true, [[TMP46]] // CHECK-NEXT: br i1 [[TMP47]], label [[RESOLVER_RETURN21:%.*]], label [[RESOLVER_ELSE22:%.*]] // CHECK: resolver_return21: @@ -581,8 +581,8 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // CHECK-NEXT: ret ptr @fmv_inline._Mfp16fmlMsimd // CHECK: resolver_else24: // CHECK-NEXT: [[TMP52:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP53:%.*]] = and i64 [[TMP52]], 16400 -// CHECK-NEXT: [[TMP54:%.*]] = icmp eq i64 [[TMP53]], 16400 +// CHECK-NEXT: [[TMP53:%.*]] = and i64 [[TMP52]], 8208 +// CHECK-NEXT: [[TMP54:%.*]] = icmp eq i64 [[TMP53]], 8208 // CHECK-NEXT: [[TMP55:%.*]] = and i1 true, [[TMP54]] // CHECK-NEXT: br i1 [[TMP55]], label [[RESOLVER_RETURN25:%.*]], label [[RESOLVER_ELSE26:%.*]] // CHECK: resolver_return25: @@ -611,8 +611,8 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 2251799813685248 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 2251799813685248 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 274877906944 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 274877906944 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: @@ -625,8 +625,8 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 70368744177664 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 70368744177664 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 34359738368 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 34359738368 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: @@ -639,8 +639,8 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 281474976710656 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 281474976710656 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 137438953472 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 137438953472 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: @@ -651,7 +651,7 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@recur -// CHECK-SAME: () #[[ATTR11]] { +// CHECK-SAME: () #[[ATTR13]] { // CHECK-NEXT: entry: // CHECK-NEXT: call void @reca() // CHECK-NEXT: ret void @@ -659,7 +659,7 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@main -// CHECK-SAME: () #[[ATTR11]] { +// CHECK-SAME: () #[[ATTR13]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[RETVAL:%.*]] = alloca i32, align 4 // CHECK-NEXT: store i32 0, ptr [[RETVAL]], align 4 @@ -670,7 +670,7 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@hoo -// CHECK-SAME: () #[[ATTR11]] { +// CHECK-SAME: () #[[ATTR13]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[FP1:%.*]] = alloca ptr, align 8 // CHECK-NEXT: [[FP2:%.*]] = alloca ptr, align 8 @@ -687,14 +687,14 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@unused_with_forward_default_decl._Mmops -// CHECK-SAME: () #[[ATTR14:[0-9]+]] { +// CHECK-SAME: () #[[ATTR7]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_extern_forward_default_decl._Mdotprod -// CHECK-SAME: () #[[ATTR15:[0-9]+]] { +// CHECK-SAME: () #[[ATTR3]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 0 // @@ -708,14 +708,14 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@unused_with_default_def._Msve -// CHECK-SAME: () #[[ATTR16:[0-9]+]] { +// CHECK-SAME: () #[[ATTR15:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@unused_with_default_def.default -// CHECK-SAME: () #[[ATTR11]] { +// CHECK-SAME: () #[[ATTR13]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 1 // @@ -729,56 +729,56 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_default_def.default -// CHECK-SAME: () #[[ATTR11]] { +// CHECK-SAME: () #[[ATTR13]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_forward_default_def.default -// CHECK-SAME: () #[[ATTR11]] { +// CHECK-SAME: () #[[ATTR13]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_forward_default_def._Mlse -// CHECK-SAME: () #[[ATTR17:[0-9]+]] { +// CHECK-SAME: () #[[ATTR16:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@unused_without_default._Mrdm -// CHECK-SAME: () #[[ATTR18:[0-9]+]] { +// CHECK-SAME: () #[[ATTR17:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@default_def_with_version_decls.default -// CHECK-SAME: () #[[ATTR11]] { +// CHECK-SAME: () #[[ATTR13]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@used_def_without_default_decl._Mjscvt -// CHECK-SAME: () #[[ATTR21:[0-9]+]] { +// CHECK-SAME: () #[[ATTR20:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@used_def_without_default_decl._Mrdm -// CHECK-SAME: () #[[ATTR18]] { +// CHECK-SAME: () #[[ATTR17]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 2 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@caller -// CHECK-SAME: () #[[ATTR11]] { +// CHECK-SAME: () #[[ATTR13]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[CALL:%.*]] = call i32 @used_def_without_default_decl() // CHECK-NEXT: [[CALL1:%.*]] = call i32 @used_decl_without_default_decl() @@ -790,8 +790,8 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1048576 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1048576 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 131072 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 131072 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: @@ -812,8 +812,8 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1048576 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1048576 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 131072 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 131072 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: @@ -831,92 +831,92 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mf64mmMpmullMsha1 -// CHECK-SAME: () #[[ATTR22:[0-9]+]] { +// CHECK-LABEL: define {{[^@]+}}@fmv_inline._McryptoMf64mm +// CHECK-SAME: () #[[ATTR21:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MfcmaMfp16MrdmMsme -// CHECK-SAME: () #[[ATTR23:[0-9]+]] { +// CHECK-SAME: () #[[ATTR22:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 2 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mf32mmMi8mmMsha3 -// CHECK-SAME: () #[[ATTR24:[0-9]+]] { +// CHECK-SAME: () #[[ATTR23:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 12 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_inline._MditMsve-ebf16 -// CHECK-SAME: () #[[ATTR25:[0-9]+]] { +// CHECK-LABEL: define {{[^@]+}}@fmv_inline._Msme2Mssbs +// CHECK-SAME: () #[[ATTR8]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 8 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MdpbMrcpc2 -// CHECK-SAME: () #[[ATTR26:[0-9]+]] { +// CHECK-SAME: () #[[ATTR24:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 6 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mdpb2Mjscvt -// CHECK-SAME: () #[[ATTR27:[0-9]+]] { +// CHECK-SAME: () #[[ATTR25:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 7 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MfrinttsMrcpc -// CHECK-SAME: () #[[ATTR28:[0-9]+]] { +// CHECK-SAME: () #[[ATTR26:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 3 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_inline._MsveMsve-bf16 -// CHECK-SAME: () #[[ATTR29:[0-9]+]] { +// CHECK-LABEL: define {{[^@]+}}@fmv_inline._MsmeMsve +// CHECK-SAME: () #[[ATTR27:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 4 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Msve2-aesMsve2-sha3 -// CHECK-SAME: () #[[ATTR30:[0-9]+]] { +// CHECK-SAME: () #[[ATTR28:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 5 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_inline._Msve2Msve2-bitpermMsve2-pmull128 -// CHECK-SAME: () #[[ATTR31:[0-9]+]] { +// CHECK-LABEL: define {{[^@]+}}@fmv_inline._Msve2Msve2-aesMsve2-bitperm +// CHECK-SAME: () #[[ATTR29:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 9 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mmemtag2Msve2-sm4 -// CHECK-SAME: () #[[ATTR32:[0-9]+]] { +// CHECK-LABEL: define {{[^@]+}}@fmv_inline._MmemtagMsve2-sm4 +// CHECK-SAME: () #[[ATTR30:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 10 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mmemtag3MmopsMrcpc3 -// CHECK-SAME: () #[[ATTR33:[0-9]+]] { +// CHECK-LABEL: define {{[^@]+}}@fmv_inline._MmemtagMmopsMrcpc3 +// CHECK-SAME: () #[[ATTR31:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 11 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MaesMdotprod -// CHECK-SAME: () #[[ATTR15]] { +// CHECK-SAME: () #[[ATTR3]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 13 // @@ -930,21 +930,21 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MfpMsm4 -// CHECK-SAME: () #[[ATTR34:[0-9]+]] { +// CHECK-SAME: () #[[ATTR32:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 15 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MlseMrdm -// CHECK-SAME: () #[[ATTR35:[0-9]+]] { +// CHECK-SAME: () #[[ATTR33:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 16 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline.default -// CHECK-SAME: () #[[ATTR11]] { +// CHECK-SAME: () #[[ATTR13]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 3 // @@ -953,8 +953,8 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1073741824 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1073741824 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 33554432 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 33554432 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: @@ -967,8 +967,8 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 65536 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 65536 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 16384 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 16384 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: @@ -995,8 +995,8 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1048576 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1048576 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 131072 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 131072 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: @@ -1132,39 +1132,37 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // CHECK: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+flagm,+fp16fml,+fullfp16,+neon,+rand,-fp-armv8,-v9.5a" } // CHECK: attributes #[[ATTR1]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+altnzcv,+bf16,+flagm,+sme,+sme-i16i64,-fp-armv8,-v9.5a" } // CHECK: attributes #[[ATTR2]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,+neon,+sha2,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR3]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+ls64,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR3]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+neon,-fp-armv8,-v9.5a" } // CHECK: attributes #[[ATTR4]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp16fml,+fullfp16,+neon,-fp-armv8,-v9.5a" } // CHECK: attributes #[[ATTR5]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,-fp-armv8,-v9.5a" } // CHECK: attributes #[[ATTR6]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR7]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bti,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR7]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,-fp-armv8,-v9.5a" } // CHECK: attributes #[[ATTR8]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme2,-fp-armv8,-v9.5a" } // CHECK: attributes #[[ATTR9:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" } // CHECK: attributes #[[ATTR10]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR11]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR11]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fptoint,-fp-armv8,-v9.5a" } // CHECK: attributes #[[ATTR12]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR13]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+sb,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR14]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR15]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+neon,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR16]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR17]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR18]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+rdm,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR19:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+jsconv,+neon,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR20:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+rdm,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR21]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+jsconv,+neon,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR22]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+aes,+f64mm,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR23]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+complxnum,+fullfp16,+neon,+rdm,+sme,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR24]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+f32mm,+fullfp16,+i8mm,+neon,+sha2,+sha3,+sve,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR25]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+dit,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR26]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,+rcpc,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR27]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccdp,+ccpp,+jsconv,+neon,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR28]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fptoint,+rcpc,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR29]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR30]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-sha3,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR31]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-bitperm,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR32]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+mte,+neon,+sve,+sve2,+sve2-sm4,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR33]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,+mte,+rcpc,+rcpc3,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR34]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+sm4,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR35]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,+neon,+rdm,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR13]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR14]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+sb,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR15]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR16]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR17]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+rdm,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR18:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+jsconv,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR19:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+rdm,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR20]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+jsconv,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR21]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+aes,+f64mm,+fullfp16,+neon,+sha2,+sve,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR22]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+complxnum,+fullfp16,+neon,+rdm,+sme,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR23]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+f32mm,+fullfp16,+i8mm,+neon,+sha2,+sha3,+sve,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR24]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,+rcpc,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR25]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccdp,+ccpp,+jsconv,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR26]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fptoint,+rcpc,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR27]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fullfp16,+neon,+sme,+sve,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR28]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-sha3,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR29]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-bitperm,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR30]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,+sve2,+sve2-sm4,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR31]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,+rcpc,+rcpc3,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR32]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+sm4,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR33]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,+neon,+rdm,-fp-armv8,-v9.5a" } //. // CHECK-NOFMV: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fmv" } // CHECK-NOFMV: attributes #[[ATTR1:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fmv" } diff --git a/clang/test/CodeGenCXX/attr-target-clones-aarch64.cpp b/clang/test/CodeGenCXX/attr-target-clones-aarch64.cpp index 7953f902bf09..d439f96982eb 100644 --- a/clang/test/CodeGenCXX/attr-target-clones-aarch64.cpp +++ b/clang/test/CodeGenCXX/attr-target-clones-aarch64.cpp @@ -1,8 +1,8 @@ // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --check-globals --include-generated-funcs // RUN: %clang_cc1 -std=c++11 -triple aarch64-linux-gnu -emit-llvm %s -o - | FileCheck %s -int __attribute__((target_clones("ls64_v+fp16", "default"))) foo_ovl(int) { return 1; } -int __attribute__((target_clones("ls64_accdata+ls64"))) foo_ovl(void) { return 2; } +int __attribute__((target_clones("ls64+fp16", "default"))) foo_ovl(int) { return 1; } +int __attribute__((target_clones("sme+ls64"))) foo_ovl(void) { return 2; } int bar() { return foo_ovl(1) + foo_ovl(); @@ -35,9 +35,6 @@ void run_foo_tml() { Mc4.foo_tml(); } - - - //. // CHECK: @__aarch64_cpu_features = external dso_local global { i64 } // CHECK: @_Z7foo_ovli.ifunc = weak_odr alias i32 (i32), ptr @_Z7foo_ovli @@ -49,7 +46,7 @@ void run_foo_tml() { // CHECK: @_ZN7MyClassIssE7foo_tmlEv = weak_odr ifunc i32 (ptr), ptr @_ZN7MyClassIssE7foo_tmlEv.resolver // CHECK: @_ZN7MyClassIisE7foo_tmlEv = weak_odr ifunc i32 (ptr), ptr @_ZN7MyClassIisE7foo_tmlEv.resolver //. -// CHECK-LABEL: @_Z7foo_ovli._Mfp16Mls64_v( +// CHECK-LABEL: @_Z7foo_ovli._Mfp16Mls64( // CHECK-NEXT: entry: // CHECK-NEXT: [[DOTADDR:%.*]] = alloca i32, align 4 // CHECK-NEXT: store i32 [[TMP0:%.*]], ptr [[DOTADDR]], align 4 @@ -60,17 +57,17 @@ void run_foo_tml() { // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 4503599627436032 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 4503599627436032 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 274877923328 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 274877923328 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: -// CHECK-NEXT: ret ptr @_Z7foo_ovli._Mfp16Mls64_v +// CHECK-NEXT: ret ptr @_Z7foo_ovli._Mfp16Mls64 // CHECK: resolver_else: // CHECK-NEXT: ret ptr @_Z7foo_ovli.default // // -// CHECK-LABEL: @_Z7foo_ovlv._Mls64Mls64_accdata( +// CHECK-LABEL: @_Z7foo_ovlv._Mls64Msme( // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 2 // @@ -79,12 +76,12 @@ void run_foo_tml() { // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 11258999068426240 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 11258999068426240 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 283467841536 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 283467841536 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: -// CHECK-NEXT: ret ptr @_Z7foo_ovlv._Mls64Mls64_accdata +// CHECK-NEXT: ret ptr @_Z7foo_ovlv._Mls64Msme // CHECK: resolver_else: // CHECK-NEXT: ret ptr @_Z7foo_ovlv.default // @@ -114,16 +111,16 @@ void run_foo_tml() { // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 36310271995674624 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 36310271995674624 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1236950581248 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1236950581248 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: // CHECK-NEXT: ret ptr @_ZN7MyClassIssE7foo_tmlEv._Msme-f64f64Mssbs // CHECK: resolver_else: // CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 16777216 -// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 16777216 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 2097152 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 2097152 // CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] // CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] // CHECK: resolver_return1: @@ -136,16 +133,16 @@ void run_foo_tml() { // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 36310271995674624 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 36310271995674624 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1236950581248 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1236950581248 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: // CHECK-NEXT: ret ptr @_ZN7MyClassIisE7foo_tmlEv._Msme-f64f64Mssbs // CHECK: resolver_else: // CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 16777216 -// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 16777216 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 2097152 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 2097152 // CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] // CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] // CHECK: resolver_return1: @@ -231,7 +228,7 @@ void run_foo_tml() { // //. // CHECK: attributes #[[ATTR0:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon" } -// CHECK: attributes #[[ATTR1:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ls64" } +// CHECK: attributes #[[ATTR1:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme" } // CHECK: attributes #[[ATTR2:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" } // CHECK: attributes #[[ATTR3:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fptoint" } // CHECK: attributes #[[ATTR4:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme-f64f64" } diff --git a/clang/test/CodeGenCXX/attr-target-version.cpp b/clang/test/CodeGenCXX/attr-target-version.cpp index 8b7273fe3bb5..24433b8f20b7 100644 --- a/clang/test/CodeGenCXX/attr-target-version.cpp +++ b/clang/test/CodeGenCXX/attr-target-version.cpp @@ -3,7 +3,7 @@ int __attribute__((target_version("sme-f64f64+bf16"))) foo(int) { return 1; } int __attribute__((target_version("default"))) foo(int) { return 2; } -int __attribute__((target_version("sm4+ebf16"))) foo(void) { return 3; } +int __attribute__((target_version("sm4+bf16"))) foo(void) { return 3; } int __attribute__((target_version("default"))) foo(void) { return 4; } struct MyClass { @@ -89,7 +89,7 @@ int bar() { // CHECK-NEXT: ret i32 2 // // -// CHECK-LABEL: @_Z3foov._Mebf16Msm4( +// CHECK-LABEL: @_Z3foov._Mbf16Msm4( // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 3 // @@ -246,8 +246,8 @@ int bar() { // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 36028797153181696 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 36028797153181696 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1099520016384 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1099520016384 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: @@ -260,12 +260,12 @@ int bar() { // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 268435488 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 268435488 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 8388640 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 8388640 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: -// CHECK-NEXT: ret ptr @_Z3foov._Mebf16Msm4 +// CHECK-NEXT: ret ptr @_Z3foov._Mbf16Msm4 // CHECK: resolver_else: // CHECK-NEXT: ret ptr @_Z3foov.default // @@ -274,8 +274,8 @@ int bar() { // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1073741824 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1073741824 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 33554432 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 33554432 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: @@ -288,8 +288,8 @@ int bar() { // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 65536 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 65536 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 16384 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 16384 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: diff --git a/clang/test/Sema/aarch64-cpu-supports.c b/clang/test/Sema/aarch64-cpu-supports.c index ddeed7c5bc9e..cf8e043dfb1e 100644 --- a/clang/test/Sema/aarch64-cpu-supports.c +++ b/clang/test/Sema/aarch64-cpu-supports.c @@ -20,7 +20,7 @@ int test_aarch64_features(void) { // expected-warning@+1 {{invalid cpu feature string}} if (__builtin_cpu_supports("default")) return 6; - if (__builtin_cpu_supports(" ssbs + bti ")) + if (__builtin_cpu_supports(" ssbs + crc ")) return 7; return 0; } diff --git a/clang/test/Sema/attr-target-clones-aarch64.c b/clang/test/Sema/attr-target-clones-aarch64.c index bc3fceab8282..511e8a941180 100644 --- a/clang/test/Sema/attr-target-clones-aarch64.c +++ b/clang/test/Sema/attr-target-clones-aarch64.c @@ -9,7 +9,7 @@ void __attribute__((target_clones("ssbs+ls64"))) warn2(void); // expected-error@+2 {{'target_clones' and 'target_version' attributes are not compatible}} // expected-note@+1 {{conflicting attribute is here}} -void __attribute__((target_version("sve-bf16"), target_clones("sme+memtag"))) not_compat(void); +void __attribute__((target_version("bf16"), target_clones("sme+memtag"))) not_compat(void); int redecl(void); int __attribute__((target_clones("frintts", "simd+fp", "default"))) redecl(void) { return 1; } @@ -21,26 +21,25 @@ int __attribute__((target_clones("sve+dotprod"))) redecl3(void); int redecl3(void); int __attribute__((target_clones("rng", "fp16fml+fp", "default"))) redecl4(void); -// expected-error@+3 {{'target_clones' attribute does not match previous declaration}} +// expected-error@+2 {{'target_clones' attribute does not match previous declaration}} // expected-note@-2 {{previous declaration is here}} -// expected-warning@+1 {{version list contains entries that don't impact code generation}} -int __attribute__((target_clones("dgh+memtag+rpres+ls64_v", "ebf16+dpb+sha1", "default"))) redecl4(void) { return 1; } +int __attribute__((target_clones("dpb2+memtag+rpres+ls64", "bf16+dpb+sha2", "default"))) redecl4(void) { return 1; } int __attribute__((target_version("flagm2"))) redef2(void) { return 1; } // expected-error@+2 {{multiversioned function redeclarations require identical target attributes}} // expected-note@-2 {{previous declaration is here}} int __attribute__((target_clones("flagm2", "default"))) redef2(void) { return 1; } -int __attribute__((target_clones("f32mm", "f64mm", "sha1+fp"))) redef3(void) { return 1; } +int __attribute__((target_clones("f32mm", "f64mm", "sha3+fp"))) redef3(void) { return 1; } // expected-error@+2 {{'target_clones' attribute does not match previous declaration}} // expected-note@-2 {{previous declaration is here}} -int __attribute__((target_clones("f32mm", "sha1+fp", "f64mm"))) redef3(void) { return 1; } +int __attribute__((target_clones("f32mm", "sha3+fp", "f64mm"))) redef3(void) { return 1; } int __attribute__((target_clones("rdm+lse+rdm", "lse+rdm"))) dup1(void) { return 1; } // expected-warning@+1 {{version list contains duplicate entries}} int __attribute__((target_clones("rdm+lse+rdm", "rdm+lse+rdm"))) dup2(void) { return 2; } // expected-warning@+1 {{version list contains duplicate entries}} -int __attribute__((target_clones("rcpc2+sve2-pmull128", "rcpc2+sve2-pmull128"))) dup3(void) { return 3; } +int __attribute__((target_clones("rcpc2+sve2-aes", "rcpc2+sve2-aes"))) dup3(void) { return 3; } // expected-warning@+1 {{version list contains duplicate entries}} void __attribute__((target_clones("sha3", "default", "default"))) dup4(void); // expected-warning@+2 {{version list contains duplicate entries}} @@ -49,7 +48,7 @@ int __attribute__((target_clones("fp", "fp", "crc+dotprod", "dotprod+crc"))) dup // expected-warning@+1 {{version list contains duplicate entries}} int __attribute__((target_clones("fp16+memtag", "memtag+fp16"))) dup6(void) { return 6; } -int __attribute__((target_clones("simd+ssbs2", "simd+dpb2"))) dup7(void) { return 7; } +int __attribute__((target_clones("simd+ssbs", "simd+dpb2"))) dup7(void) { return 7; } // expected-warning@+1 {{unsupported '' in the 'target_clones' attribute string;}} void __attribute__((target_clones(""))) empty_target_1(void); @@ -72,13 +71,13 @@ empty_target_5(void); void __attribute__((target_clones("sve2-bitperm", "sve2-bitperm"))) dupe_normal(void); -void __attribute__((target_clones("default"), target_clones("memtag3+bti"))) dupe_normal2(void); +void __attribute__((target_clones("default"), target_clones("memtag+mops"))) dupe_normal2(void); int mv_after_use(void); int useage(void) { return mv_after_use(); } // expected-error@+1 {{function declaration cannot become a multiversioned function after first usage}} -int __attribute__((target_clones("sve2-sha3+ssbs2", "sm4"))) mv_after_use(void) { return 1; } +int __attribute__((target_clones("sve2-sha3+ssbs", "sm4"))) mv_after_use(void) { return 1; } // expected-error@+1 {{'main' cannot be a multiversioned function}} -int __attribute__((target_clones("sve-i8mm"))) main() { return 1; } +int __attribute__((target_clones("sve2-aes"))) main() { return 1; } diff --git a/clang/test/Sema/attr-target-version.c b/clang/test/Sema/attr-target-version.c index cd5be459456e..4ff8e1eee619 100644 --- a/clang/test/Sema/attr-target-version.c +++ b/clang/test/Sema/attr-target-version.c @@ -16,7 +16,7 @@ int __attribute__((target_version("aes"))) foo(void) { return 1; } int __attribute__((target_version("default"))) foo(void) { return 2; } //expected-note@+1 {{previous definition is here}} -int __attribute__((target_version("sha3 + pmull "))) foo(void) { return 1; } +int __attribute__((target_version("sha3 + aes "))) foo(void) { return 1; } //expected-note@-1 {{previous definition is here}} //expected-error@+1 {{redefinition of 'foo'}} @@ -32,11 +32,11 @@ __attribute__ ((target("bf16,sve,sve2,dotprod"))) int func(void) { return 1; } __attribute__ ((target("default"))) int func(void) { return 0; } //expected-note@+1 {{previous declaration is here}} -void __attribute__((target_version("bti+flagm2"))) one(void) {} +void __attribute__((target_version("mops+flagm2"))) one(void) {} //expected-error@+1 {{multiversioned function redeclarations require identical target attributes}} -void __attribute__((target_version("flagm2+bti"))) one(void) {} +void __attribute__((target_version("flagm2+mops"))) one(void) {} -void __attribute__((target_version("ssbs+sha1"))) two(void) {} +void __attribute__((target_version("ssbs+sha2"))) two(void) {} void __attribute__((target_version("ssbs+fp16fml"))) two(void) {} //expected-error@+1 {{'main' cannot be a multiversioned function}} @@ -44,7 +44,7 @@ int __attribute__((target_version("lse"))) main(void) { return 1; } // It is ok for the default version to appear first. int default_first(void) { return 1; } -int __attribute__((target_version("dit"))) default_first(void) { return 2; } +int __attribute__((target_version("crc"))) default_first(void) { return 2; } int __attribute__((target_version("mops"))) default_first(void) { return 3; } // It is ok if the default version is between other versions. @@ -77,7 +77,7 @@ void __attribute__((target_version("rdm+rng+crc"))) redef(void) {} void __attribute__((target_version("rdm+rng+crc"))) redef(void) {} int def(void); -void __attribute__((target_version("dit"))) nodef(void); +void __attribute__((target_version("crc"))) nodef(void); void __attribute__((target_version("ls64"))) nodef(void); void __attribute__((target_version("aes"))) ovl(void); void __attribute__((target_version("default"))) ovl(void); @@ -89,12 +89,12 @@ int bar() { return def(); } // expected-error@+1 {{function declaration cannot become a multiversioned function after first usage}} -int __attribute__((target_version("sha1"))) def(void) { return 1; } +int __attribute__((target_version("sha3"))) def(void) { return 1; } int __attribute__((target_version("sve"))) prot(); // expected-error@-1 {{multiversioned function must have a prototype}} -int __attribute__((target_version("pmull"))) rtype(int); +int __attribute__((target_version("aes"))) rtype(int); // expected-error@+1 {{multiversioned function declaration has a different return type}} float __attribute__((target_version("rdm"))) rtype(int); @@ -102,7 +102,7 @@ int __attribute__((target_version("sha2"))) combine(void) { return 1; } // expected-error@+1 {{multiversioned function declaration has a different calling convention}} int __attribute__((aarch64_vector_pcs, target_version("sha3"))) combine(void) { return 2; } -int __attribute__((target_version("fp+aes+pmull+rcpc"))) unspec_args() { return -1; } +int __attribute__((target_version("fp+aes+sha3+rcpc"))) unspec_args() { return -1; } // expected-error@-1 {{multiversioned function must have a prototype}} // expected-error@+1 {{multiversioned function must have a prototype}} int __attribute__((target_version("default"))) unspec_args() { return 0; } diff --git a/clang/test/SemaCXX/attr-target-version.cpp b/clang/test/SemaCXX/attr-target-version.cpp index b3385f043590..ae2923cc303c 100644 --- a/clang/test/SemaCXX/attr-target-version.cpp +++ b/clang/test/SemaCXX/attr-target-version.cpp @@ -25,13 +25,13 @@ int __attribute__((target_version("dpb"))) diff_link(void); int __attribute__((target_version("memtag"))) diff_link1(void) { return 1; } //expected-error@+1 {{multiversioned function declaration has a different linkage}} -static int __attribute__((target_version("bti"))) diff_link1(void); +static int __attribute__((target_version("mops"))) diff_link1(void); int __attribute__((target_version("flagm2"))) diff_link2(void) { return 1; } extern int __attribute__((target_version("flagm"))) diff_link2(void); namespace { -static int __attribute__((target_version("memtag3"))) diff_link2(void) { return 2; } +static int __attribute__((target_version("memtag"))) diff_link2(void) { return 2; } int __attribute__((target_version("sve2-bitperm"))) diff_link2(void) { return 1; } } // namespace @@ -49,7 +49,7 @@ double __attribute__((target_version("rcpc"))) diff_type1(void); auto __attribute__((target_version("rcpc2"))) diff_type2(void) -> int { return 1; } //expected-error@+1 {{multiversioned function declaration has a different return type}} -auto __attribute__((target_version("sve-bf16"))) diff_type2(void) -> long { return (long)1; } +auto __attribute__((target_version("sve2-aes"))) diff_type2(void) -> long { return (long)1; } int __attribute__((target_version("fp16fml"))) diff_type3(void) noexcept(false) { return 1; } //expected-error@+2 {{exception specification in declaration does not match previous declaration}} @@ -75,7 +75,7 @@ auto __attribute__((target_version("dpb2"))) ret3(void) -> int { return 1; } class Cls { __attribute__((target_version("rng"))) Cls(); // expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support constructors}} - __attribute__((target_version("sve-i8mm"))) ~Cls(); + __attribute__((target_version("sve2-aes"))) ~Cls(); // expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support destructors}} Cls &__attribute__((target_version("f32mm"))) operator=(const Cls &) = default; @@ -98,11 +98,11 @@ __attribute__((target_version("jscvt"))) void Decl(); } // namespace Nms class Out { - int __attribute__((target_version("bti"))) func(void); - int __attribute__((target_version("ssbs2"))) func(void); + int __attribute__((target_version("mops"))) func(void); + int __attribute__((target_version("ssbs"))) func(void); }; -int __attribute__((target_version("bti"))) Out::func(void) { return 1; } -int __attribute__((target_version("ssbs2"))) Out::func(void) { return 2; } +int __attribute__((target_version("mops"))) Out::func(void) { return 1; } +int __attribute__((target_version("ssbs"))) Out::func(void) { return 2; } // expected-error@+3 {{out-of-line definition of 'func' does not match any declaration in 'Out'}} // expected-note@-3 {{member declaration nearly matches}} // expected-note@-3 {{member declaration nearly matches}} diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64.c b/compiler-rt/lib/builtins/cpu_model/aarch64.c index 17bddfca46f0..1ac4d85a0c13 100644 --- a/compiler-rt/lib/builtins/cpu_model/aarch64.c +++ b/compiler-rt/lib/builtins/cpu_model/aarch64.c @@ -67,13 +67,10 @@ enum CPUFeatures { FEAT_FP, FEAT_SIMD, FEAT_CRC, - FEAT_SHA1, FEAT_SHA2, FEAT_SHA3, FEAT_AES, - FEAT_PMULL, FEAT_FP16, - FEAT_DIT, FEAT_DPB, FEAT_DPB2, FEAT_JSCVT, @@ -81,35 +78,23 @@ enum CPUFeatures { FEAT_RCPC, FEAT_RCPC2, FEAT_FRINTTS, - FEAT_DGH, FEAT_I8MM, FEAT_BF16, - FEAT_EBF16, FEAT_RPRES, FEAT_SVE, - FEAT_SVE_BF16, - FEAT_SVE_EBF16, - FEAT_SVE_I8MM, FEAT_SVE_F32MM, FEAT_SVE_F64MM, FEAT_SVE2, FEAT_SVE_AES, - FEAT_SVE_PMULL128, FEAT_SVE_BITPERM, FEAT_SVE_SHA3, FEAT_SVE_SM4, FEAT_SME, FEAT_MEMTAG, - FEAT_MEMTAG2, - FEAT_MEMTAG3, FEAT_SB, FEAT_PREDRES, FEAT_SSBS, - FEAT_SSBS2, - FEAT_BTI, FEAT_LS64, - FEAT_LS64_V, - FEAT_LS64_ACCDATA, FEAT_WFXT, FEAT_SME_F64, FEAT_SME_I64, @@ -117,7 +102,7 @@ enum CPUFeatures { FEAT_RCPC3, FEAT_MOPS, FEAT_MAX, - FEAT_EXT = 62, // Reserved to indicate presence of additional features field + FEAT_EXT = 47, // Reserved to indicate presence of additional features field // in __aarch64_cpu_features FEAT_INIT // Used as flag of features initialization completion }; diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/apple.inc b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/apple.inc index 6fef109567b6..19114c4abdfc 100644 --- a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/apple.inc +++ b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/apple.inc @@ -35,13 +35,10 @@ void __init_cpu_features_resolver(void) { {"hw.optional.floatingpoint", FEAT_FP}, {"hw.optional.AdvSIMD", FEAT_SIMD}, {"hw.optional.armv8_crc32", FEAT_CRC}, - {"hw.optional.arm.FEAT_SHA1", FEAT_SHA1}, {"hw.optional.arm.FEAT_SHA256", FEAT_SHA2}, {"hw.optional.arm.FEAT_SHA3", FEAT_SHA3}, {"hw.optional.arm.FEAT_AES", FEAT_AES}, - {"hw.optional.arm.FEAT_PMULL", FEAT_PMULL}, {"hw.optional.arm.FEAT_FP16", FEAT_FP16}, - {"hw.optional.arm.FEAT_DIT", FEAT_DIT}, {"hw.optional.arm.FEAT_DPB", FEAT_DPB}, {"hw.optional.arm.FEAT_DPB2", FEAT_DPB2}, {"hw.optional.arm.FEAT_JSCVT", FEAT_JSCVT}, @@ -53,8 +50,7 @@ void __init_cpu_features_resolver(void) { {"hw.optional.arm.FEAT_BF16", FEAT_BF16}, {"hw.optional.arm.FEAT_SB", FEAT_SB}, {"hw.optional.arm.FEAT_SPECRES", FEAT_PREDRES}, - {"hw.optional.arm.FEAT_SSBS", FEAT_SSBS2}, - {"hw.optional.arm.FEAT_BTI", FEAT_BTI}, + {"hw.optional.arm.FEAT_SSBS", FEAT_SSBS}, }; for (size_t I = 0, E = sizeof(feature_checks) / sizeof(feature_checks[0]); diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/fuchsia.inc b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/fuchsia.inc index d8e0280f4041..579aa00dc5c7 100644 --- a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/fuchsia.inc +++ b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/fuchsia.inc @@ -22,10 +22,6 @@ void __init_cpu_features_resolver() { setCPUFeature(FEAT_SIMD); if (features & ZX_ARM64_FEATURE_ISA_AES) setCPUFeature(FEAT_AES); - if (features & ZX_ARM64_FEATURE_ISA_PMULL) - setCPUFeature(FEAT_PMULL); - if (features & ZX_ARM64_FEATURE_ISA_SHA1) - setCPUFeature(FEAT_SHA1); if (features & ZX_ARM64_FEATURE_ISA_SHA256) setCPUFeature(FEAT_SHA2); if (features & ZX_ARM64_FEATURE_ISA_CRC32) diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/mrs.inc b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/mrs.inc index 32a21a2fba9a..54797852e4aa 100644 --- a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/mrs.inc +++ b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/mrs.inc @@ -16,8 +16,6 @@ static void __init_cpu_features_constructor(unsigned long hwcap, hwcap2 = arg->_hwcap2; if (hwcap & HWCAP_CRC32) setCPUFeature(FEAT_CRC); - if (hwcap & HWCAP_PMULL) - setCPUFeature(FEAT_PMULL); if (hwcap & HWCAP_FLAGM) setCPUFeature(FEAT_FLAGM); if (hwcap2 & HWCAP2_FLAGM2) { @@ -34,16 +32,12 @@ static void __init_cpu_features_constructor(unsigned long hwcap, setCPUFeature(FEAT_FP16); setCPUFeature(FEAT_FP); } - if (hwcap & HWCAP_DIT) - setCPUFeature(FEAT_DIT); if (hwcap & HWCAP_ASIMDRDM) setCPUFeature(FEAT_RDM); if (hwcap & HWCAP_ILRCPC) setCPUFeature(FEAT_RCPC2); if (hwcap & HWCAP_AES) setCPUFeature(FEAT_AES); - if (hwcap & HWCAP_SHA1) - setCPUFeature(FEAT_SHA1); if (hwcap & HWCAP_SHA2) setCPUFeature(FEAT_SHA2); if (hwcap & HWCAP_JSCVT) @@ -53,22 +47,11 @@ static void __init_cpu_features_constructor(unsigned long hwcap, if (hwcap & HWCAP_SB) setCPUFeature(FEAT_SB); if (hwcap & HWCAP_SSBS) - setCPUFeature(FEAT_SSBS2); - if (hwcap2 & HWCAP2_MTE) { + setCPUFeature(FEAT_SSBS); + if (hwcap2 & HWCAP2_MTE) setCPUFeature(FEAT_MEMTAG); - setCPUFeature(FEAT_MEMTAG2); - } - if (hwcap2 & HWCAP2_MTE3) { - setCPUFeature(FEAT_MEMTAG); - setCPUFeature(FEAT_MEMTAG2); - setCPUFeature(FEAT_MEMTAG3); - } if (hwcap2 & HWCAP2_SVEAES) setCPUFeature(FEAT_SVE_AES); - if (hwcap2 & HWCAP2_SVEPMULL) { - setCPUFeature(FEAT_SVE_AES); - setCPUFeature(FEAT_SVE_PMULL128); - } if (hwcap2 & HWCAP2_SVEBITPERM) setCPUFeature(FEAT_SVE_BITPERM); if (hwcap2 & HWCAP2_SVESHA3) @@ -83,22 +66,12 @@ static void __init_cpu_features_constructor(unsigned long hwcap, setCPUFeature(FEAT_RNG); if (hwcap2 & HWCAP2_I8MM) setCPUFeature(FEAT_I8MM); - if (hwcap2 & HWCAP2_EBF16) - setCPUFeature(FEAT_EBF16); - if (hwcap2 & HWCAP2_SVE_EBF16) - setCPUFeature(FEAT_SVE_EBF16); - if (hwcap2 & HWCAP2_DGH) - setCPUFeature(FEAT_DGH); if (hwcap2 & HWCAP2_FRINT) setCPUFeature(FEAT_FRINTTS); - if (hwcap2 & HWCAP2_SVEI8MM) - setCPUFeature(FEAT_SVE_I8MM); if (hwcap2 & HWCAP2_SVEF32MM) setCPUFeature(FEAT_SVE_F32MM); if (hwcap2 & HWCAP2_SVEF64MM) setCPUFeature(FEAT_SVE_F64MM); - if (hwcap2 & HWCAP2_BTI) - setCPUFeature(FEAT_BTI); if (hwcap2 & HWCAP2_RPRES) setCPUFeature(FEAT_RPRES); if (hwcap2 & HWCAP2_WFXT) @@ -141,9 +114,6 @@ static void __init_cpu_features_constructor(unsigned long hwcap, // ID_AA64ZFR0_EL1.SVEver == 0b0001 if (extractBits(ftr, 0, 4) == 0x1) setCPUFeature(FEAT_SVE2); - // ID_AA64ZFR0_EL1.BF16 != 0b0000 - if (extractBits(ftr, 20, 4) != 0x0) - setCPUFeature(FEAT_SVE_BF16); } getCPUFeature(ID_AA64ISAR0_EL1, ftr); // ID_AA64ISAR0_EL1.SHA3 != 0b0000 @@ -168,12 +138,6 @@ static void __init_cpu_features_constructor(unsigned long hwcap, // ID_AA64ISAR1_EL1.LS64 >= 0b0001 if (extractBits(ftr, 60, 4) >= 0x1) setCPUFeature(FEAT_LS64); - // ID_AA64ISAR1_EL1.LS64 >= 0b0010 - if (extractBits(ftr, 60, 4) >= 0x2) - setCPUFeature(FEAT_LS64_V); - // ID_AA64ISAR1_EL1.LS64 >= 0b0011 - if (extractBits(ftr, 60, 4) >= 0x3) - setCPUFeature(FEAT_LS64_ACCDATA); } else { // Set some features in case of no CPUID support if (hwcap & (HWCAP_FP | HWCAP_FPHP)) { @@ -187,8 +151,6 @@ static void __init_cpu_features_constructor(unsigned long hwcap, setCPUFeature(FEAT_RCPC); if (hwcap2 & HWCAP2_BF16 || hwcap2 & HWCAP2_EBF16) setCPUFeature(FEAT_BF16); - if (hwcap2 & HWCAP2_SVEBF16) - setCPUFeature(FEAT_SVE_BF16); if (hwcap2 & HWCAP2_SVE2 && hwcap & HWCAP_SVE) setCPUFeature(FEAT_SVE2); if (hwcap & HWCAP_SHA3) diff --git a/llvm/include/llvm/TargetParser/AArch64TargetParser.h b/llvm/include/llvm/TargetParser/AArch64TargetParser.h index 805b963a7a13..bcb36beb3cc0 100644 --- a/llvm/include/llvm/TargetParser/AArch64TargetParser.h +++ b/llvm/include/llvm/TargetParser/AArch64TargetParser.h @@ -46,13 +46,10 @@ enum CPUFeatures { FEAT_FP, FEAT_SIMD, FEAT_CRC, - FEAT_SHA1, FEAT_SHA2, FEAT_SHA3, FEAT_AES, - FEAT_PMULL, FEAT_FP16, - FEAT_DIT, FEAT_DPB, FEAT_DPB2, FEAT_JSCVT, @@ -60,35 +57,23 @@ enum CPUFeatures { FEAT_RCPC, FEAT_RCPC2, FEAT_FRINTTS, - FEAT_DGH, FEAT_I8MM, FEAT_BF16, - FEAT_EBF16, FEAT_RPRES, FEAT_SVE, - FEAT_SVE_BF16, - FEAT_SVE_EBF16, - FEAT_SVE_I8MM, FEAT_SVE_F32MM, FEAT_SVE_F64MM, FEAT_SVE2, FEAT_SVE_AES, - FEAT_SVE_PMULL128, FEAT_SVE_BITPERM, FEAT_SVE_SHA3, FEAT_SVE_SM4, FEAT_SME, FEAT_MEMTAG, - FEAT_MEMTAG2, - FEAT_MEMTAG3, FEAT_SB, FEAT_PREDRES, FEAT_SSBS, - FEAT_SSBS2, - FEAT_BTI, FEAT_LS64, - FEAT_LS64_V, - FEAT_LS64_ACCDATA, FEAT_WFXT, FEAT_SME_F64, FEAT_SME_I64, @@ -96,12 +81,12 @@ enum CPUFeatures { FEAT_RCPC3, FEAT_MOPS, FEAT_MAX, - FEAT_EXT = 62, + FEAT_EXT = 47, FEAT_INIT }; -static_assert(FEAT_MAX < 62, - "Number of features in CPUFeatures are limited to 62 entries"); +static_assert(FEAT_MAX < 47, + "Number of features in CPUFeatures are limited to 47 entries"); // Arch extension modifiers for CPUs. These are labelled with their Arm ARM // feature name (though the canonical reference for those is AArch64.td) @@ -215,17 +200,13 @@ inline constexpr ExtensionInfo Extensions[] = { {"b16b16", AArch64::AEK_B16B16, "+b16b16", "-b16b16", FEAT_INIT, "", 0}, {"bf16", AArch64::AEK_BF16, "+bf16", "-bf16", FEAT_BF16, "+bf16", 280}, {"brbe", AArch64::AEK_BRBE, "+brbe", "-brbe", FEAT_INIT, "", 0}, - {"bti", AArch64::AEK_NONE, {}, {}, FEAT_BTI, "+bti", 510}, {"crc", AArch64::AEK_CRC, "+crc", "-crc", FEAT_CRC, "+crc", 110}, {"crypto", AArch64::AEK_CRYPTO, "+crypto", "-crypto", FEAT_INIT, "+aes,+sha2", 0}, {"cssc", AArch64::AEK_CSSC, "+cssc", "-cssc", FEAT_INIT, "", 0}, {"d128", AArch64::AEK_D128, "+d128", "-d128", FEAT_INIT, "", 0}, - {"dgh", AArch64::AEK_NONE, {}, {}, FEAT_DGH, "", 260}, - {"dit", AArch64::AEK_NONE, {}, {}, FEAT_DIT, "+dit", 180}, {"dotprod", AArch64::AEK_DOTPROD, "+dotprod", "-dotprod", FEAT_DOTPROD, "+dotprod,+fp-armv8,+neon", 104}, {"dpb", AArch64::AEK_NONE, {}, {}, FEAT_DPB, "+ccpp", 190}, {"dpb2", AArch64::AEK_NONE, {}, {}, FEAT_DPB2, "+ccpp,+ccdp", 200}, - {"ebf16", AArch64::AEK_NONE, {}, {}, FEAT_EBF16, "+bf16", 290}, {"f32mm", AArch64::AEK_F32MM, "+f32mm", "-f32mm", FEAT_SVE_F32MM, "+sve,+f32mm,+fullfp16,+fp-armv8,+neon", 350}, {"f64mm", AArch64::AEK_F64MM, "+f64mm", "-f64mm", FEAT_SVE_F64MM, "+sve,+f64mm,+fullfp16,+fp-armv8,+neon", 360}, {"fcma", AArch64::AEK_FCMA, "+complxnum", "-complxnum", FEAT_FCMA, "+fp-armv8,+neon,+complxnum", 220}, @@ -239,17 +220,12 @@ inline constexpr ExtensionInfo Extensions[] = { {"i8mm", AArch64::AEK_I8MM, "+i8mm", "-i8mm", FEAT_I8MM, "+i8mm", 270}, {"ite", AArch64::AEK_ITE, "+ite", "-ite", FEAT_INIT, "", 0}, {"jscvt", AArch64::AEK_JSCVT, "+jsconv", "-jsconv", FEAT_JSCVT, "+fp-armv8,+neon,+jsconv", 210}, - {"ls64_accdata", AArch64::AEK_NONE, {}, {}, FEAT_LS64_ACCDATA, "+ls64", 540}, - {"ls64_v", AArch64::AEK_NONE, {}, {}, FEAT_LS64_V, "", 530}, {"ls64", AArch64::AEK_LS64, "+ls64", "-ls64", FEAT_LS64, "", 520}, {"lse", AArch64::AEK_LSE, "+lse", "-lse", FEAT_LSE, "+lse", 80}, {"lse128", AArch64::AEK_LSE128, "+lse128", "-lse128", FEAT_INIT, "", 0}, {"memtag", AArch64::AEK_MTE, "+mte", "-mte", FEAT_MEMTAG, "", 440}, - {"memtag2", AArch64::AEK_NONE, {}, {}, FEAT_MEMTAG2, "+mte", 450}, - {"memtag3", AArch64::AEK_NONE, {}, {}, FEAT_MEMTAG3, "+mte", 460}, {"mops", AArch64::AEK_MOPS, "+mops", "-mops", FEAT_MOPS, "+mops", 650}, {"pauth", AArch64::AEK_PAUTH, "+pauth", "-pauth", FEAT_INIT, "", 0}, - {"pmull", AArch64::AEK_NONE, {}, {}, FEAT_PMULL, "+aes,+fp-armv8,+neon", 160}, {"pmuv3", AArch64::AEK_PERFMON, "+perfmon", "-perfmon", FEAT_INIT, "", 0}, {"predres", AArch64::AEK_PREDRES, "+predres", "-predres", FEAT_PREDRES, "+predres", 480}, {"predres2", AArch64::AEK_SPECRES2, "+specres2", "-specres2", FEAT_INIT, "", 0}, @@ -263,7 +239,6 @@ inline constexpr ExtensionInfo Extensions[] = { {"rng", AArch64::AEK_RAND, "+rand", "-rand", FEAT_RNG, "+rand", 10}, {"rpres", AArch64::AEK_NONE, {}, {}, FEAT_RPRES, "", 300}, {"sb", AArch64::AEK_SB, "+sb", "-sb", FEAT_SB, "+sb", 470}, - {"sha1", AArch64::AEK_NONE, {}, {}, FEAT_SHA1, "+fp-armv8,+neon", 120}, {"sha2", AArch64::AEK_SHA2, "+sha2", "-sha2", FEAT_SHA2, "+sha2,+fp-armv8,+neon", 130}, {"sha3", AArch64::AEK_SHA3, "+sha3", "-sha3", FEAT_SHA3, "+sha3,+sha2,+fp-armv8,+neon", 140}, {"simd", AArch64::AEK_SIMD, "+neon", "-neon", FEAT_SIMD, "+fp-armv8,+neon", 100}, @@ -275,14 +250,9 @@ inline constexpr ExtensionInfo Extensions[] = { {"sme2", AArch64::AEK_SME2, "+sme2", "-sme2", FEAT_SME2, "+sme2,+sme,+bf16", 580}, {"sme2p1", AArch64::AEK_SME2p1, "+sme2p1", "-sme2p1", FEAT_INIT, "+sme2p1,+sme2,+sme,+bf16", 0}, {"ssbs", AArch64::AEK_SSBS, "+ssbs", "-ssbs", FEAT_SSBS, "", 490}, - {"ssbs2", AArch64::AEK_NONE, {}, {}, FEAT_SSBS2, "+ssbs", 500}, - {"sve-bf16", AArch64::AEK_NONE, {}, {}, FEAT_SVE_BF16, "+sve,+bf16,+fullfp16,+fp-armv8,+neon", 320}, - {"sve-ebf16", AArch64::AEK_NONE, {}, {}, FEAT_SVE_EBF16, "+sve,+bf16,+fullfp16,+fp-armv8,+neon", 330}, - {"sve-i8mm", AArch64::AEK_NONE, {}, {}, FEAT_SVE_I8MM, "+sve,+i8mm,+fullfp16,+fp-armv8,+neon", 340}, {"sve", AArch64::AEK_SVE, "+sve", "-sve", FEAT_SVE, "+sve,+fullfp16,+fp-armv8,+neon", 310}, {"sve2-aes", AArch64::AEK_SVE2AES, "+sve2-aes", "-sve2-aes", FEAT_SVE_AES, "+sve2,+sve,+sve2-aes,+fullfp16,+fp-armv8,+neon", 380}, {"sve2-bitperm", AArch64::AEK_SVE2BITPERM, "+sve2-bitperm", "-sve2-bitperm", FEAT_SVE_BITPERM, "+sve2,+sve,+sve2-bitperm,+fullfp16,+fp-armv8,+neon", 400}, - {"sve2-pmull128", AArch64::AEK_NONE, {}, {}, FEAT_SVE_PMULL128, "+sve2,+sve,+sve2-aes,+fullfp16,+fp-armv8,+neon", 390}, {"sve2-sha3", AArch64::AEK_SVE2SHA3, "+sve2-sha3", "-sve2-sha3", FEAT_SVE_SHA3, "+sve2,+sve,+sve2-sha3,+fullfp16,+fp-armv8,+neon", 410}, {"sve2-sm4", AArch64::AEK_SVE2SM4, "+sve2-sm4", "-sve2-sm4", FEAT_SVE_SM4, "+sve2,+sve,+sve2-sm4,+fullfp16,+fp-armv8,+neon", 420}, {"sve2", AArch64::AEK_SVE2, "+sve2", "-sve2", FEAT_SVE2, "+sve2,+sve,+fullfp16,+fp-armv8,+neon", 370}, -- GitLab From 06947b9f8d258fe66fc69f1e7c0197cb621da3a5 Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Wed, 17 Apr 2024 09:31:29 -0700 Subject: [PATCH 072/862] [libc][POSIX][pthreads] implement pthread_condattr_t functions (#88987) Implement: - pthread_condattr_destroy - pthread_condattr_getclock - pthread_condattr_getpshared - pthread_condattr_init - pthread_condattr_setclock - pthread_condattr_setpshared Fixes: #88581 --- libc/config/linux/api.td | 29 ++++++-- libc/config/linux/x86_64/entrypoints.txt | 6 ++ libc/include/CMakeLists.txt | 1 + libc/include/llvm-libc-types/CMakeLists.txt | 3 +- .../llvm-libc-types/pthread_condattr_t.h | 18 +++++ libc/include/pthread.h.def | 6 ++ libc/spec/posix.td | 64 ++++++++++++++++- libc/src/pthread/CMakeLists.txt | 65 +++++++++++++++++ libc/src/pthread/pthread_condattr_destroy.cpp | 23 ++++++ libc/src/pthread/pthread_condattr_destroy.h | 20 ++++++ .../src/pthread/pthread_condattr_getclock.cpp | 25 +++++++ libc/src/pthread/pthread_condattr_getclock.h | 22 ++++++ .../pthread/pthread_condattr_getpshared.cpp | 24 +++++++ .../src/pthread/pthread_condattr_getpshared.h | 21 ++++++ libc/src/pthread/pthread_condattr_init.cpp | 24 +++++++ libc/src/pthread/pthread_condattr_init.h | 20 ++++++ .../src/pthread/pthread_condattr_setclock.cpp | 30 ++++++++ libc/src/pthread/pthread_condattr_setclock.h | 21 ++++++ .../pthread/pthread_condattr_setpshared.cpp | 28 ++++++++ .../src/pthread/pthread_condattr_setpshared.h | 20 ++++++ libc/test/src/pthread/CMakeLists.txt | 12 ++++ .../src/pthread/pthread_condattr_test.cpp | 71 +++++++++++++++++++ 22 files changed, 545 insertions(+), 8 deletions(-) create mode 100644 libc/include/llvm-libc-types/pthread_condattr_t.h create mode 100644 libc/src/pthread/pthread_condattr_destroy.cpp create mode 100644 libc/src/pthread/pthread_condattr_destroy.h create mode 100644 libc/src/pthread/pthread_condattr_getclock.cpp create mode 100644 libc/src/pthread/pthread_condattr_getclock.h create mode 100644 libc/src/pthread/pthread_condattr_getpshared.cpp create mode 100644 libc/src/pthread/pthread_condattr_getpshared.h create mode 100644 libc/src/pthread/pthread_condattr_init.cpp create mode 100644 libc/src/pthread/pthread_condattr_init.h create mode 100644 libc/src/pthread/pthread_condattr_setclock.cpp create mode 100644 libc/src/pthread/pthread_condattr_setclock.h create mode 100644 libc/src/pthread/pthread_condattr_setpshared.cpp create mode 100644 libc/src/pthread/pthread_condattr_setpshared.h create mode 100644 libc/test/src/pthread/pthread_condattr_test.cpp diff --git a/libc/config/linux/api.td b/libc/config/linux/api.td index 9964971f191b..5fb92a9c299c 100644 --- a/libc/config/linux/api.td +++ b/libc/config/linux/api.td @@ -175,6 +175,7 @@ def PThreadAPI : PublicAPI<"pthread.h"> { "__pthread_start_t", "__pthread_tss_dtor_t", "pthread_attr_t", + "pthread_condattr_t", "pthread_mutex_t", "pthread_mutexattr_t", "pthread_t", @@ -241,10 +242,30 @@ def SysSendfileAPI : PublicAPI<"sys/sendfile.h"> { } def SysTypesAPI : PublicAPI<"sys/types.h"> { - let Types = ["blkcnt_t", "blksize_t", "clockid_t", "dev_t", "gid_t", "ino_t", - "mode_t", "nlink_t", "off_t", "pid_t", "pthread_attr_t", "pthread_key_t", - "pthread_mutex_t", "pthread_mutexattr_t", "pthread_once_t", "pthread_t", - "size_t", "ssize_t", "suseconds_t", "time_t", "uid_t"]; + let Types = [ + "blkcnt_t", + "blksize_t", + "clockid_t", + "dev_t", + "gid_t", + "ino_t", + "mode_t", + "nlink_t", + "off_t", + "pid_t", + "pthread_attr_t", + "pthread_condattr_t", + "pthread_key_t", + "pthread_mutex_t", + "pthread_mutexattr_t", + "pthread_once_t", + "pthread_t", + "size_t", + "ssize_t", + "suseconds_t", + "time_t", + "uid_t" + ]; } def SysUtsNameAPI : PublicAPI<"sys/utsname.h"> { diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt index 70f130a4399a..2d8136536b21 100644 --- a/libc/config/linux/x86_64/entrypoints.txt +++ b/libc/config/linux/x86_64/entrypoints.txt @@ -639,6 +639,12 @@ if(LLVM_LIBC_FULL_BUILD) libc.src.pthread.pthread_attr_setguardsize libc.src.pthread.pthread_attr_setstack libc.src.pthread.pthread_attr_setstacksize + libc.src.pthread.pthread_condattr_destroy + libc.src.pthread.pthread_condattr_getclock + libc.src.pthread.pthread_condattr_getpshared + libc.src.pthread.pthread_condattr_init + libc.src.pthread.pthread_condattr_setclock + libc.src.pthread.pthread_condattr_setpshared libc.src.pthread.pthread_create libc.src.pthread.pthread_detach libc.src.pthread.pthread_equal diff --git a/libc/include/CMakeLists.txt b/libc/include/CMakeLists.txt index b85366c8deaf..f5ba2791af3f 100644 --- a/libc/include/CMakeLists.txt +++ b/libc/include/CMakeLists.txt @@ -321,6 +321,7 @@ add_gen_header( .llvm-libc-types.__pthread_start_t .llvm-libc-types.__pthread_tss_dtor_t .llvm-libc-types.pthread_attr_t + .llvm-libc-types.pthread_condattr_t .llvm-libc-types.pthread_mutex_t .llvm-libc-types.pthread_mutexattr_t .llvm-libc-types.pthread_t diff --git a/libc/include/llvm-libc-types/CMakeLists.txt b/libc/include/llvm-libc-types/CMakeLists.txt index 93a79e1477b3..f26fc0729dc9 100644 --- a/libc/include/llvm-libc-types/CMakeLists.txt +++ b/libc/include/llvm-libc-types/CMakeLists.txt @@ -49,11 +49,12 @@ add_header(pid_t HDR pid_t.h) add_header(posix_spawn_file_actions_t HDR posix_spawn_file_actions_t.h) add_header(posix_spawnattr_t HDR posix_spawnattr_t.h) add_header(pthread_attr_t HDR pthread_attr_t.h DEPENDS .size_t) +add_header(pthread_condattr_t HDR pthread_condattr_t.h DEPENDS .clockid_t) add_header(pthread_key_t HDR pthread_key_t.h) add_header(pthread_mutex_t HDR pthread_mutex_t.h DEPENDS .__futex_word .__mutex_type) -add_header(pthread_t HDR pthread_t.h DEPENDS .__thread_type) add_header(pthread_mutexattr_t HDR pthread_mutexattr_t.h) add_header(pthread_once_t HDR pthread_once_t.h DEPENDS .__futex_word) +add_header(pthread_t HDR pthread_t.h DEPENDS .__thread_type) add_header(rlim_t HDR rlim_t.h) add_header(time_t HDR time_t.h) add_header(stack_t HDR stack_t.h) diff --git a/libc/include/llvm-libc-types/pthread_condattr_t.h b/libc/include/llvm-libc-types/pthread_condattr_t.h new file mode 100644 index 000000000000..b91fc2950aa3 --- /dev/null +++ b/libc/include/llvm-libc-types/pthread_condattr_t.h @@ -0,0 +1,18 @@ +//===-- Definition of pthread_condattr_t type -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_LIBC_TYPES_PTHREAD_CONDATTR_T_H +#define LLVM_LIBC_TYPES_PTHREAD_CONDATTR_T_H + +#include "clockid_t.h" + +typedef struct { + clockid_t clock; + int pshared; +} pthread_condattr_t; + +#endif // LLVM_LIBC_TYPES_PTHREAD_CONDATTR_T_H diff --git a/libc/include/pthread.h.def b/libc/include/pthread.h.def index abeb839ee83d..a94d770657e1 100644 --- a/libc/include/pthread.h.def +++ b/libc/include/pthread.h.def @@ -11,6 +11,9 @@ #include "__llvm-libc-common.h" +// TODO: move to a pthreads-macros.h file: +// https://github.com/llvm/llvm-project/issues/88997 + #define PTHREAD_STACK_MIN (1 << 14) // 16KB #define PTHREAD_MUTEX_INITIALIZER {0} @@ -32,6 +35,9 @@ enum { PTHREAD_MUTEX_ROBUST = 0x1, }; +#define PTHREAD_PROCESS_PRIVATE 0 +#define PTHREAD_PROCESS_SHARED 1 + %%public_api() #endif // LLVM_LIBC_PTHREAD_H diff --git a/libc/spec/posix.td b/libc/spec/posix.td index 7095a3964ee3..0c88dbd848a3 100644 --- a/libc/spec/posix.td +++ b/libc/spec/posix.td @@ -26,6 +26,7 @@ def UidT : NamedType<"uid_t">; def GidT : NamedType<"gid_t">; def DevT : NamedType<"dev_t">; def ClockIdT : NamedType<"clockid_t">; +def RestrictedClockIdTPtr : RestrictedPtrType; def BlkSizeT : NamedType<"blksize_t">; def BlkCntT : NamedType<"blkcnt_t">; def NLinkT : NamedType<"nlink_t">; @@ -105,6 +106,10 @@ def POSIX : StandardSpec<"POSIX"> { ConstType ConstPThreadAttrTPtr = ConstType; ConstType ConstRestrictedPThreadAttrTPtr = ConstType; + NamedType PThreadCondAttrTType = NamedType<"pthread_condattr_t">; + PtrType PThreadCondAttrTPtr = PtrType; + ConstType ConstRestrictedPThreadCondAttrTPtr = ConstType>; + NamedType PThreadMutexAttrTType = NamedType<"pthread_mutexattr_t">; PtrType PThreadMutexAttrTPtr = PtrType; RestrictedPtrType RestrictedPThreadMutexAttrTPtr = RestrictedPtrType; @@ -980,7 +985,9 @@ def POSIX : StandardSpec<"POSIX"> { [], // Macros [ AtForkCallbackT, + ClockIdT, PThreadAttrTType, + PThreadCondAttrTType, PThreadKeyT, PThreadMutexAttrTType, PThreadMutexTType, @@ -1047,6 +1054,36 @@ def POSIX : StandardSpec<"POSIX"> { RetValSpec, [ArgSpec, ArgSpec, ArgSpec] >, + FunctionSpec< + "pthread_condattr_destroy", + RetValSpec, + [ArgSpec] + >, + FunctionSpec< + "pthread_condattr_getclock", + RetValSpec, + [ArgSpec, ArgSpec] + >, + FunctionSpec< + "pthread_condattr_getpshared", + RetValSpec, + [ArgSpec, ArgSpec] + >, + FunctionSpec< + "pthread_condattr_init", + RetValSpec, + [ArgSpec] + >, + FunctionSpec< + "pthread_condattr_setclock", + RetValSpec, + [ArgSpec, ArgSpec] + >, + FunctionSpec< + "pthread_condattr_setpshared", + RetValSpec, + [ArgSpec, ArgSpec] + >, FunctionSpec< "pthread_create", RetValSpec, @@ -1522,9 +1559,30 @@ def POSIX : StandardSpec<"POSIX"> { HeaderSpec SysTypes = HeaderSpec< "sys/types.h", [], // Macros - [BlkCntT, BlkSizeT, ClockIdT, DevT, GidT, InoT, ModeTType, NLinkT, OffTType, PidT, - PThreadAttrTType, PThreadKeyT, PThreadMutexTType, PThreadMutexAttrTType, PThreadOnceT, PThreadTType, - SizeTType, SSizeTType, SuSecondsT, TimeTType, UidT], + [ + BlkCntT, + BlkSizeT, + ClockIdT, + DevT, + GidT, + InoT, + ModeTType, + NLinkT, + OffTType, + PThreadAttrTType, + PThreadCondAttrTType, + PThreadKeyT, + PThreadMutexAttrTType, + PThreadMutexTType, + PThreadOnceT, + PThreadTType, + PidT, + SSizeTType, + SizeTType, + SuSecondsT, + TimeTType, + UidT + ], // Types [], // Enumerations [] // Functions >; diff --git a/libc/src/pthread/CMakeLists.txt b/libc/src/pthread/CMakeLists.txt index d5e6c802a845..3d6cf6dde010 100644 --- a/libc/src/pthread/CMakeLists.txt +++ b/libc/src/pthread/CMakeLists.txt @@ -100,6 +100,71 @@ add_entrypoint_object( libc.src.pthread.pthread_attr_setstacksize ) +add_entrypoint_object( + pthread_condattr_destroy + SRCS + pthread_condattr_destroy.cpp + HDRS + pthread_condattr_destroy.h + DEPENDS + libc.include.pthread +) + +add_entrypoint_object( + pthread_condattr_getclock + SRCS + pthread_condattr_getclock.cpp + HDRS + pthread_condattr_getclock.h + DEPENDS + libc.include.pthread + libc.include.sys_types +) + +add_entrypoint_object( + pthread_condattr_getpshared + SRCS + pthread_condattr_getpshared.cpp + HDRS + pthread_condattr_getpshared.h + DEPENDS + libc.include.pthread +) + +add_entrypoint_object( + pthread_condattr_init + SRCS + pthread_condattr_init.cpp + HDRS + pthread_condattr_init.h + DEPENDS + libc.include.pthread + libc.include.time +) + +add_entrypoint_object( + pthread_condattr_setclock + SRCS + pthread_condattr_setclock.cpp + HDRS + pthread_condattr_setclock.h + DEPENDS + libc.include.errno + libc.include.pthread + libc.include.sys_types + libc.include.time +) + +add_entrypoint_object( + pthread_condattr_setpshared + SRCS + pthread_condattr_setpshared.cpp + HDRS + pthread_condattr_setpshared.h + DEPENDS + libc.include.pthread +) + add_header_library( pthread_mutexattr HDRS diff --git a/libc/src/pthread/pthread_condattr_destroy.cpp b/libc/src/pthread/pthread_condattr_destroy.cpp new file mode 100644 index 000000000000..45cc011a4a92 --- /dev/null +++ b/libc/src/pthread/pthread_condattr_destroy.cpp @@ -0,0 +1,23 @@ +//===-- Implementation of the pthread_condattr_destroy --------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "pthread_condattr_destroy.h" + +#include "src/__support/common.h" + +#include + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(int, pthread_condattr_destroy, (pthread_condattr_t * attr)) { + // Initializing a pthread_condattr_t acquires no resources, so this is a + // no-op. + return 0; +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/pthread/pthread_condattr_destroy.h b/libc/src/pthread/pthread_condattr_destroy.h new file mode 100644 index 000000000000..2910fa9f9616 --- /dev/null +++ b/libc/src/pthread/pthread_condattr_destroy.h @@ -0,0 +1,20 @@ +//===-- Implementation header for pthread_condattr_destroy ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_DESTROY_H +#define LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_DESTROY_H + +#include + +namespace LIBC_NAMESPACE { + +int pthread_condattr_destroy(pthread_condattr_t *attr); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_DESTROY_H diff --git a/libc/src/pthread/pthread_condattr_getclock.cpp b/libc/src/pthread/pthread_condattr_getclock.cpp new file mode 100644 index 000000000000..a3a3963f4f42 --- /dev/null +++ b/libc/src/pthread/pthread_condattr_getclock.cpp @@ -0,0 +1,25 @@ +//===-- Implementation of the pthread_condattr_getclock -------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "pthread_condattr_getclock.h" + +#include "src/__support/common.h" + +#include // pthread_condattr_t +#include // clockid_t + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(int, pthread_condattr_getclock, + (const pthread_condattr_t *__restrict attr, + clockid_t *__restrict clock_id)) { + *clock_id = attr->clock; + return 0; +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/pthread/pthread_condattr_getclock.h b/libc/src/pthread/pthread_condattr_getclock.h new file mode 100644 index 000000000000..d5878c4f45b5 --- /dev/null +++ b/libc/src/pthread/pthread_condattr_getclock.h @@ -0,0 +1,22 @@ +//===-- Implementation header for pthread_condattr_getclock -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_GETCLOCK_H +#define LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_GETCLOCK_H + +#include // pthread_condattr_t +#include // clockid_t + +namespace LIBC_NAMESPACE { + +int pthread_condattr_getclock(const pthread_condattr_t *__restrict attr, + clockid_t *__restrict clock_id); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_GETCLOCK_H diff --git a/libc/src/pthread/pthread_condattr_getpshared.cpp b/libc/src/pthread/pthread_condattr_getpshared.cpp new file mode 100644 index 000000000000..0c5fdc115c25 --- /dev/null +++ b/libc/src/pthread/pthread_condattr_getpshared.cpp @@ -0,0 +1,24 @@ +//===-- Implementation of the pthread_condattr_getpshared -----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "pthread_condattr_getpshared.h" + +#include "src/__support/common.h" + +#include + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(int, pthread_condattr_getpshared, + (const pthread_condattr_t *__restrict attr, + int *__restrict pshared)) { + *pshared = attr->pshared; + return 0; +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/pthread/pthread_condattr_getpshared.h b/libc/src/pthread/pthread_condattr_getpshared.h new file mode 100644 index 000000000000..3d7a0c1d357c --- /dev/null +++ b/libc/src/pthread/pthread_condattr_getpshared.h @@ -0,0 +1,21 @@ +//===-- Implementation header for pthread_condattr_getpshared ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_PSHARED_H +#define LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_PSHARED_H + +#include + +namespace LIBC_NAMESPACE { + +int pthread_condattr_getpshared(const pthread_condattr_t *__restrict attr, + int *__restrict pshared); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_PSHARED_H diff --git a/libc/src/pthread/pthread_condattr_init.cpp b/libc/src/pthread/pthread_condattr_init.cpp new file mode 100644 index 000000000000..54633b2e3a5e --- /dev/null +++ b/libc/src/pthread/pthread_condattr_init.cpp @@ -0,0 +1,24 @@ +//===-- Implementation of the pthread_condattr_init -----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "pthread_condattr_init.h" + +#include "src/__support/common.h" + +#include // pthread_condattr_t, PTHREAD_PROCESS_PRIVATE +#include // CLOCK_REALTIME + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(int, pthread_condattr_init, (pthread_condattr_t * attr)) { + attr->clock = CLOCK_REALTIME; + attr->pshared = PTHREAD_PROCESS_PRIVATE; + return 0; +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/pthread/pthread_condattr_init.h b/libc/src/pthread/pthread_condattr_init.h new file mode 100644 index 000000000000..9f3c06bb6f4a --- /dev/null +++ b/libc/src/pthread/pthread_condattr_init.h @@ -0,0 +1,20 @@ +//===-- Implementation header for pthread_condattr_init ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_INIT_H +#define LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_INIT_H + +#include + +namespace LIBC_NAMESPACE { + +int pthread_condattr_init(pthread_condattr_t *attr); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_INIT_H diff --git a/libc/src/pthread/pthread_condattr_setclock.cpp b/libc/src/pthread/pthread_condattr_setclock.cpp new file mode 100644 index 000000000000..6eca8b30ef7f --- /dev/null +++ b/libc/src/pthread/pthread_condattr_setclock.cpp @@ -0,0 +1,30 @@ +//===-- Implementation of the pthread_condattr_setclock -------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "pthread_condattr_setclock.h" + +#include "src/__support/common.h" + +#include // EINVAL +#include // pthread_condattr_t +#include // clockid_t +#include // CLOCK_MONOTONIC, CLOCK_REALTIME + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(int, pthread_condattr_setclock, + (pthread_condattr_t * attr, clockid_t clock)) { + + if (clock != CLOCK_MONOTONIC && clock != CLOCK_REALTIME) + return EINVAL; + + attr->clock = clock; + return 0; +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/pthread/pthread_condattr_setclock.h b/libc/src/pthread/pthread_condattr_setclock.h new file mode 100644 index 000000000000..328766fe7883 --- /dev/null +++ b/libc/src/pthread/pthread_condattr_setclock.h @@ -0,0 +1,21 @@ +//===-- Implementation header for pthread_condattr_setclock -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_SETCLOCK_H +#define LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_SETCLOCK_H + +#include +#include // clockid_t + +namespace LIBC_NAMESPACE { + +int pthread_condattr_setclock(pthread_condattr_t *attr, clockid_t clock); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_SETCLOCK_H diff --git a/libc/src/pthread/pthread_condattr_setpshared.cpp b/libc/src/pthread/pthread_condattr_setpshared.cpp new file mode 100644 index 000000000000..7f1560acad84 --- /dev/null +++ b/libc/src/pthread/pthread_condattr_setpshared.cpp @@ -0,0 +1,28 @@ +//===-- Implementation of the pthread_condattr_setpshared -----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "pthread_condattr_setpshared.h" + +#include "src/__support/common.h" + +#include // EINVAL +#include // pthread_condattr_t, PTHREAD_PROCESS_SHARED, PTHREAD_PROCESS_PRIVATE + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(int, pthread_condattr_setpshared, + (pthread_condattr_t * attr, int pshared)) { + + if (pshared != PTHREAD_PROCESS_SHARED && pshared != PTHREAD_PROCESS_PRIVATE) + return EINVAL; + + attr->pshared = pshared; + return 0; +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/pthread/pthread_condattr_setpshared.h b/libc/src/pthread/pthread_condattr_setpshared.h new file mode 100644 index 000000000000..8083bdec78cc --- /dev/null +++ b/libc/src/pthread/pthread_condattr_setpshared.h @@ -0,0 +1,20 @@ +//===-- Implementation header for pthread_condattr_setpshared ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_SETPSHARED_H +#define LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_SETPSHARED_H + +#include + +namespace LIBC_NAMESPACE { + +int pthread_condattr_setpshared(pthread_condattr_t *attr, int pshared); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_SETPSHARED_H diff --git a/libc/test/src/pthread/CMakeLists.txt b/libc/test/src/pthread/CMakeLists.txt index fb0d22ab9d2a..46f38422a537 100644 --- a/libc/test/src/pthread/CMakeLists.txt +++ b/libc/test/src/pthread/CMakeLists.txt @@ -39,3 +39,15 @@ add_libc_unittest( libc.src.pthread.pthread_mutexattr_setrobust libc.src.pthread.pthread_mutexattr_settype ) + +add_libc_unittest( + pthread_condattr_test + SUITE + libc_pthread_unittests + SRCS + pthread_condattr_test.cpp + DEPENDS + libc.include.errno + libc.include.pthread + libc.include.time + ) diff --git a/libc/test/src/pthread/pthread_condattr_test.cpp b/libc/test/src/pthread/pthread_condattr_test.cpp new file mode 100644 index 000000000000..accb62de92e4 --- /dev/null +++ b/libc/test/src/pthread/pthread_condattr_test.cpp @@ -0,0 +1,71 @@ +//===-- Unittests for pthread_condattr_t ----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "test/UnitTest/Test.h" + +#include +#include +#include + +TEST(LlvmLibcPThreadCondAttrTest, InitAndDestroy) { + pthread_condattr_t cond; + ASSERT_EQ(pthread_condattr_init(&cond), 0); + ASSERT_EQ(pthread_condattr_destroy(&cond), 0); +} + +TEST(LlvmLibcPThreadCondAttrTest, GetDefaultValues) { + pthread_condattr_t cond; + + // Invalid clock id. + clockid_t clock = 7; + // Invalid value. + int pshared = 42; + + ASSERT_EQ(pthread_condattr_init(&cond), 0); + ASSERT_EQ(pthread_condattr_getclock(&cond, &clock), 0); + ASSERT_EQ(clock, CLOCK_REALTIME); + ASSERT_EQ(pthread_condattr_getpshared(&cond, &pshared), 0); + ASSERT_EQ(pshared, PTHREAD_PROCESS_PRIVATE); + ASSERT_EQ(pthread_condattr_destroy(&cond), 0); +} + +TEST(LlvmLibcPThreadCondAttrTest, SetGoodValues) { + pthread_condattr_t cond; + + // Invalid clock id. + clockid_t clock = 7; + // Invalid value. + int pshared = 42; + + ASSERT_EQ(pthread_condattr_init(&cond), 0); + ASSERT_EQ(pthread_condattr_setclock(&cond, CLOCK_MONOTONIC), 0); + ASSERT_EQ(pthread_condattr_getclock(&cond, &clock), 0); + ASSERT_EQ(clock, CLOCK_MONOTONIC); + ASSERT_EQ(pthread_condattr_setpshared(&cond, PTHREAD_PROCESS_SHARED), 0); + ASSERT_EQ(pthread_condattr_getpshared(&cond, &pshared), 0); + ASSERT_EQ(pshared, PTHREAD_PROCESS_SHARED); + ASSERT_EQ(pthread_condattr_destroy(&cond), 0); +} + +TEST(LlvmLibcPThreadCondAttrTest, SetBadValues) { + pthread_condattr_t cond; + + // Invalid clock id. + clockid_t clock = 7; + // Invalid value. + int pshared = 42; + + ASSERT_EQ(pthread_condattr_init(&cond), 0); + ASSERT_EQ(pthread_condattr_setclock(&cond, clock), EINVAL); + ASSERT_EQ(pthread_condattr_getclock(&cond, &clock), 0); + ASSERT_EQ(clock, CLOCK_REALTIME); + ASSERT_EQ(pthread_condattr_setpshared(&cond, pshared), EINVAL); + ASSERT_EQ(pthread_condattr_getpshared(&cond, &pshared), 0); + ASSERT_EQ(pshared, PTHREAD_PROCESS_PRIVATE); + ASSERT_EQ(pthread_condattr_destroy(&cond), 0); +} -- GitLab From 4edeaffbf255137861f5153eb1a6183d956efede Mon Sep 17 00:00:00 2001 From: fabrizio-indirli Date: Wed, 17 Apr 2024 17:43:22 +0100 Subject: [PATCH 073/862] [mlir][tosa] Fix tosa.Resize-to-linalg lowering (#88514) --- .../Conversion/TosaToLinalg/TosaToLinalg.cpp | 21 ++-- .../TosaToLinalg/tosa-to-linalg-resize.mlir | 112 ++++++++---------- 2 files changed, 60 insertions(+), 73 deletions(-) diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp index d8dd1c93722b..af19ebaea937 100644 --- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp +++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp @@ -1582,17 +1582,16 @@ public: } // x = x * scale_d + offset; // ix = floor(x / scale_n) - // dx = x / scale_n - ix - Value val = b.create(floatTy, in); - scaleN = b.create(floatTy, scaleN); - scaleD = b.create(floatTy, scaleD); - offset = b.create(floatTy, offset); - val = b.create(val, scaleD); - val = b.create(val, offset); - val = b.create(val, scaleN); - index = b.create(val); - delta = b.create(val, index); - index = b.create(b.getI32Type(), index); + Value val = b.create(in, scaleD); + val = b.create(val, offset); + index = b.create(val, scaleN); + + // rx = x % scale_n + // dx = rx / scale_n + Value r = b.create(val, scaleN); + Value rFp = b.create(floatTy, r); + Value scaleNfp = b.create(floatTy, scaleN); + delta = b.create(rFp, scaleNfp); }; // Compute the ix and dx values for the X and Y dimensions - int case. diff --git a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-resize.mlir b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-resize.mlir index 468e92e2a266..d42d0a46692d 100644 --- a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-resize.mlir +++ b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-resize.mlir @@ -304,42 +304,36 @@ func.func @resize_nearest_fp32(%input: tensor<1x50x48x1xf32>) -> () { // CHECK-DAG: %[[XMAX:.*]] = arith.constant 47 // CHECK: %[[Y:.+]] = arith.index_cast %[[IDX1]] // CHECK: %[[X:.+]] = arith.index_cast %[[IDX2]] - // CHECK-DAG: %[[ISCALE_Y_N:.*]] = arith.constant 64 - // CHECK-DAG: %[[ISCALE_Y_D:.*]] = arith.constant 2 - // CHECK-DAG: %[[ISCALE_X_N:.*]] = arith.constant 64 - // CHECK-DAG: %[[ISCALE_X_D:.*]] = arith.constant 2 - // CHECK-DAG: %[[IOFFSET_Y:.*]] = arith.constant -31 - // CHECK-DAG: %[[IOFFSET_X:.*]] = arith.constant -31 - // CHECK-DAG: %[[IBORDER_Y:.*]] = arith.constant 31 - // CHECK-DAG: %[[IBORDER_X:.*]] = arith.constant 31 - - // CHECK: %[[Y0:.+]] = arith.uitofp %[[Y]] - // CHECK: %[[SCALE_Y_N:.*]] = arith.uitofp %[[ISCALE_Y_N]] - // CHECK: %[[SCALE_Y_D:.*]] = arith.uitofp %[[ISCALE_Y_D]] - // CHECK: %[[OFFSET_Y:.*]] = arith.sitofp %[[IOFFSET_Y]] - // CHECK: %[[VAL_29:.*]] = arith.mulf %[[Y0]], %[[SCALE_Y_D]] - // CHECK: %[[VAL_31:.*]] = arith.addf %[[VAL_29]], %[[OFFSET_Y]] - // CHECK: %[[VAL_33:.*]] = arith.divf %[[VAL_31]], %[[SCALE_Y_N]] - // CHECK: %[[VAL_35:.*]] = math.floor %[[VAL_33]] - // CHECK: %[[D_Y:.*]] = arith.subf %[[VAL_33]], %[[VAL_35]] - // CHECK: %[[VAL_39:.*]] = arith.fptosi %[[VAL_35]] - - // CHECK: %[[X0:.+]] = arith.uitofp %[[X]] - // CHECK: %[[SCALE_X_N:.*]] = arith.uitofp %[[ISCALE_X_N]] - // CHECK: %[[SCALE_X_D:.*]] = arith.uitofp %[[ISCALE_X_D]] - // CHECK: %[[OFFSET_X:.*]] = arith.sitofp %[[IOFFSET_X]] - // CHECK: %[[VAL_30:.*]] = arith.mulf %[[X0]], %[[SCALE_X_D]] - // CHECK: %[[VAL_32:.*]] = arith.addf %[[VAL_30]], %[[OFFSET_X]] - // CHECK: %[[VAL_34:.*]] = arith.divf %[[VAL_32]], %[[SCALE_X_N]] - // CHECK: %[[VAL_36:.*]] = math.floor %[[VAL_34]] - // CHECK: %[[D_X:.*]] = arith.subf %[[VAL_34]], %[[VAL_36]] - // CHECK: %[[VAL_40:.*]] = arith.fptosi %[[VAL_36]] + // CHECK-DAG: %[[SCALE_Y_N:.*]] = arith.constant 64 + // CHECK-DAG: %[[SCALE_Y_D:.*]] = arith.constant 2 + // CHECK-DAG: %[[SCALE_X_N:.*]] = arith.constant 64 + // CHECK-DAG: %[[SCALE_X_D:.*]] = arith.constant 2 + // CHECK-DAG: %[[OFFSET_Y:.*]] = arith.constant -31 + // CHECK-DAG: %[[OFFSET_X:.*]] = arith.constant -31 + // CHECK-DAG: %[[BORDER_Y:.*]] = arith.constant 31 + // CHECK-DAG: %[[BORDER_X:.*]] = arith.constant 31 + + // CHECK: %[[VAL_29:.*]] = arith.muli %[[Y]], %[[SCALE_Y_D]] + // CHECK: %[[Y_TEMP:.*]] = arith.addi %[[VAL_29]], %[[OFFSET_Y]] + // CHECK: %[[IY_TEMP:.*]] = arith.floordivsi %[[Y_TEMP]], %[[SCALE_Y_N]] + // CHECK: %[[RY:.*]] = arith.remsi %[[Y_TEMP]], %[[SCALE_Y_N]] + // CHECK: %[[RY_FP:.*]] = arith.sitofp %[[RY]] + // CHECK: %[[SCALE_Y_N_FP:.*]] = arith.uitofp %[[SCALE_Y_N]] + // CHECK: %[[D_Y:.*]] = arith.divf %[[RY_FP]], %[[SCALE_Y_N_FP]] + + // CHECK: %[[VAL_30:.*]] = arith.muli %[[X]], %[[SCALE_X_D]] + // CHECK: %[[X_TEMP:.*]] = arith.addi %[[VAL_30]], %[[OFFSET_X]] + // CHECK: %[[IX_TEMP:.*]] = arith.floordivsi %[[X_TEMP]], %[[SCALE_X_N]] + // CHECK: %[[RX:.*]] = arith.remsi %[[X_TEMP]], %[[SCALE_X_N]] + // CHECK: %[[RX_FP:.*]] = arith.sitofp %[[RX]] + // CHECK: %[[SCALE_X_N_FP:.*]] = arith.uitofp %[[SCALE_X_N]] + // CHECK: %[[D_X:.*]] = arith.divf %[[RX_FP]], %[[SCALE_X_N_FP]] // CHECK-DAG: %[[ONE:.*]] = arith.constant 1 // CHECK-DAG: %[[HALF:.*]] = arith.constant 5.000000e-01 // CHECK: %[[PRED_Y:.*]] = arith.cmpf oge, %[[D_Y]], %[[HALF]] // CHECK: %[[ROUND_Y:.*]] = arith.select %[[PRED_Y]], %[[ONE]], %[[ZERO]] - // CHECK: %[[VAL_48:.*]] = arith.addi %[[VAL_39]], %[[ROUND_Y]] + // CHECK: %[[VAL_48:.*]] = arith.addi %[[IY_TEMP]], %[[ROUND_Y]] // CHECK: %[[LOWER:.*]] = arith.maxsi %[[ZERO]], %[[VAL_48]] // CHECK: %[[CLAMPED:.*]] = arith.minsi %[[YMAX]], %[[LOWER]] // CHECK: %[[IDY:.*]] = arith.index_cast %[[CLAMPED]] @@ -347,7 +341,7 @@ func.func @resize_nearest_fp32(%input: tensor<1x50x48x1xf32>) -> () { // CHECK-DAG: %[[HALF:.*]] = arith.constant 5.000000e-01 // CHECK: %[[PRED_X:.*]] = arith.cmpf oge, %[[D_X]], %[[HALF]] // CHECK: %[[ROUND_X:.*]] = arith.select %[[PRED_X]], %[[ONE]], %[[ZERO]] - // CHECK: %[[VAL_49:.*]] = arith.addi %[[VAL_40]], %[[ROUND_X]] + // CHECK: %[[VAL_49:.*]] = arith.addi %[[IX_TEMP]], %[[ROUND_X]] // CHECK: %[[LOWER:.*]] = arith.maxsi %[[ZERO]], %[[VAL_49]] // CHECK: %[[CLAMPED:.*]] = arith.minsi %[[XMAX]], %[[LOWER]] // CHECK: %[[IDX:.*]] = arith.index_cast %[[CLAMPED]] @@ -374,36 +368,30 @@ func.func @resize_bilinear_fp(%input: tensor<1x23x24x1xf32>) -> () { // CHECK-DAG: %[[X_MAX:.*]] = arith.constant 23 // CHECK: %[[Y:.+]] = arith.index_cast %[[IDX_1]] // CHECK: %[[X:.+]] = arith.index_cast %[[IDX_2]] - // CHECK-DAG: %[[ISCALE_Y_N:.*]] = arith.constant 4 - // CHECK-DAG: %[[ISCALE_Y_D:.*]] = arith.constant 1 - // CHECK-DAG: %[[ISCALE_X_N:.*]] = arith.constant 4 - // CHECK-DAG: %[[ISCALE_X_D:.*]] = arith.constant 1 - // CHECK-DAG: %[[IOFFSET_Y:.*]] = arith.constant 0 - // CHECK-DAG: %[[IOFFSET_X:.*]] = arith.constant 0 - // CHECK-DAG: %[[IBORDER_Y:.*]] = arith.constant 0 - // CHECK-DAG: %[[IBORDER_X:.*]] = arith.constant 0 - - // CHECK: %[[Y0:.+]] = arith.uitofp %[[Y]] - // CHECK: %[[SCALE_Y_N:.*]] = arith.uitofp %[[ISCALE_Y_N]] - // CHECK: %[[SCALE_Y_D:.*]] = arith.uitofp %[[ISCALE_Y_D]] - // CHECK: %[[OFFSET_Y:.*]] = arith.sitofp %[[IOFFSET_Y]] - // CHECK: %[[VAL_29:.*]] = arith.mulf %[[Y0]], %[[SCALE_Y_D]] - // CHECK: %[[VAL_31:.*]] = arith.addf %[[VAL_29]], %[[OFFSET_Y]] - // CHECK: %[[VAL_33:.*]] = arith.divf %[[VAL_31]], %[[SCALE_Y_N]] - // CHECK: %[[VAL_35:.*]] = math.floor %[[VAL_33]] - // CHECK: %[[D_Y:.*]] = arith.subf %[[VAL_33]], %[[VAL_35]] - // CHECK: %[[I_Y:.*]] = arith.fptosi %[[VAL_35]] - - // CHECK: %[[X0:.+]] = arith.uitofp %[[X]] - // CHECK: %[[SCALE_X_N:.*]] = arith.uitofp %[[ISCALE_X_N]] - // CHECK: %[[SCALE_X_D:.*]] = arith.uitofp %[[ISCALE_X_D]] - // CHECK: %[[OFFSET_X:.*]] = arith.sitofp %[[IOFFSET_X]] - // CHECK: %[[VAL_30:.*]] = arith.mulf %[[X0]], %[[SCALE_X_D]] - // CHECK: %[[VAL_32:.*]] = arith.addf %[[VAL_30]], %[[OFFSET_X]] - // CHECK: %[[VAL_34:.*]] = arith.divf %[[VAL_32]], %[[SCALE_X_N]] - // CHECK: %[[VAL_36:.*]] = math.floor %[[VAL_34]] - // CHECK: %[[D_X:.*]] = arith.subf %[[VAL_34]], %[[VAL_36]] - // CHECK: %[[I_X:.*]] = arith.fptosi %[[VAL_36]] + // CHECK-DAG: %[[SCALE_Y_N:.*]] = arith.constant 4 + // CHECK-DAG: %[[SCALE_Y_D:.*]] = arith.constant 1 + // CHECK-DAG: %[[SCALE_X_N:.*]] = arith.constant 4 + // CHECK-DAG: %[[SCALE_X_D:.*]] = arith.constant 1 + // CHECK-DAG: %[[OFFSET_Y:.*]] = arith.constant 0 + // CHECK-DAG: %[[OFFSET_X:.*]] = arith.constant 0 + // CHECK-DAG: %[[BORDER_Y:.*]] = arith.constant 0 + // CHECK-DAG: %[[BORDER_X:.*]] = arith.constant 0 + + // CHECK: %[[VAL_29:.*]] = arith.muli %[[Y]], %[[SCALE_Y_D]] + // CHECK: %[[Y_TEMP:.*]] = arith.addi %[[VAL_29]], %[[OFFSET_Y]] + // CHECK: %[[I_Y:.*]] = arith.floordivsi %[[Y_TEMP]], %[[SCALE_Y_N]] + // CHECK: %[[RY:.*]] = arith.remsi %[[Y_TEMP]], %[[SCALE_Y_N]] + // CHECK: %[[RY_FP:.*]] = arith.sitofp %[[RY]] + // CHECK: %[[SCALE_Y_N_FP:.*]] = arith.uitofp %[[SCALE_Y_N]] + // CHECK: %[[D_Y:.*]] = arith.divf %[[RY_FP]], %[[SCALE_Y_N_FP]] + + // CHECK: %[[VAL_30:.*]] = arith.muli %[[X]], %[[SCALE_X_D]] + // CHECK: %[[X_TEMP:.*]] = arith.addi %[[VAL_30]], %[[OFFSET_X]] + // CHECK: %[[I_X:.*]] = arith.floordivsi %[[X_TEMP]], %[[SCALE_X_N]] + // CHECK: %[[RX:.*]] = arith.remsi %[[X_TEMP]], %[[SCALE_X_N]] + // CHECK: %[[RX_FP:.*]] = arith.sitofp %[[RX]] + // CHECK: %[[SCALE_X_N_FP:.*]] = arith.uitofp %[[SCALE_X_N]] + // CHECK: %[[D_X:.*]] = arith.divf %[[RX_FP]], %[[SCALE_X_N_FP]] // Compute the left, right, and top indices for the bilinear interpolation. -- GitLab From 564f9abfcc3b99a01843f88b5a2c7309bfab5a33 Mon Sep 17 00:00:00 2001 From: Jorge Gorbe Moya Date: Wed, 17 Apr 2024 10:11:03 -0700 Subject: [PATCH 074/862] [bazel][mlir] Add missing dep after 4f88c2311130791cf69da34b743b1b3ba7584a7b --- utils/bazel/llvm-project-overlay/mlir/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index 03386549a011..58538b66c5e0 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -577,6 +577,7 @@ mlir_c_api_cc_library( ], includes = ["include"], deps = [ + ":IR", ":NVGPUDialect", ], ) -- GitLab From e59632bdfd4c70caed437216af17d335858686fd Mon Sep 17 00:00:00 2001 From: Michael Maitland Date: Wed, 17 Apr 2024 06:19:11 -0700 Subject: [PATCH 075/862] [RISCV] Fix typo in RISCVScheduleV.td that was introduced in 60a1158 --- llvm/lib/Target/RISCV/RISCVScheduleV.td | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Target/RISCV/RISCVScheduleV.td b/llvm/lib/Target/RISCV/RISCVScheduleV.td index 5993884bc2c1..5be06d4c3f7e 100644 --- a/llvm/lib/Target/RISCV/RISCVScheduleV.td +++ b/llvm/lib/Target/RISCV/RISCVScheduleV.td @@ -1079,7 +1079,7 @@ defm "" : LMULReadAdvance<"ReadVFCvtFToIV", 0>; defm "" : LMULSEWReadAdvanceW<"ReadVFWCvtIToFV", 0>; defm "" : LMULReadAdvanceFW<"ReadVFWCvtFToIV", 0>; defm "" : LMULSEWReadAdvanceFW<"ReadVFWCvtFToFV", 0>; -defm "" : LMULSEWReadAdvanceFW<"SEWReadVFNCvtIToFV", 0>; +defm "" : LMULSEWReadAdvanceFW<"ReadVFNCvtIToFV", 0>; defm "" : LMULReadAdvanceW<"ReadVFNCvtFToIV", 0>; defm "" : LMULSEWReadAdvanceFW<"ReadVFNCvtFToFV", 0>; -- GitLab From 676d3bafc09d0c331a04b813804407334de12917 Mon Sep 17 00:00:00 2001 From: Jorge Gorbe Moya Date: Wed, 17 Apr 2024 10:20:07 -0700 Subject: [PATCH 076/862] [bazel][libc] Add missing dep after b854a2323337be2633b1135f590678a17e9d1ade --- utils/bazel/llvm-project-overlay/libc/BUILD.bazel | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel index 6029cc3fee61..be02c2270432 100644 --- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel @@ -786,7 +786,7 @@ libc_support_library( ":errno", ":hdr_fenv_macros", ":hdr_math_macros", - ":types_fenv_t" + ":types_fenv_t", ], ) @@ -1152,6 +1152,7 @@ libc_function( deps = [ ":__support_common", ":__support_fputil_fenv_impl", + ":types_fexcept_t", ], ) @@ -1272,7 +1273,7 @@ libc_function( deps = [ ":__support_common", ":__support_fputil_fenv_impl", - ":types_fexcept_t" + ":types_fexcept_t", ], ) @@ -1283,7 +1284,7 @@ libc_function( deps = [ ":__support_common", ":__support_fputil_fenv_impl", - ":types_fexcept_t", + ":types_fexcept_t", ], ) -- GitLab From 693a458287d019c5c6a66fe3019d099df2978cdb Mon Sep 17 00:00:00 2001 From: Abdul Raheem <55028856+abdulraheembeigh@users.noreply.github.com> Date: Wed, 17 Apr 2024 18:24:04 +0100 Subject: [PATCH 077/862] [MLIR] Update doc comment in ViewLikeInterface.td (NFC) (#89074) Signed-off: Abdul Raheem Beigh --- mlir/include/mlir/Interfaces/ViewLikeInterface.td | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlir/include/mlir/Interfaces/ViewLikeInterface.td b/mlir/include/mlir/Interfaces/ViewLikeInterface.td index ea5bb1b5ac48..9397f271e1bc 100644 --- a/mlir/include/mlir/Interfaces/ViewLikeInterface.td +++ b/mlir/include/mlir/Interfaces/ViewLikeInterface.td @@ -158,7 +158,7 @@ def OffsetSizeAndStrideOpInterface : OpInterface<"OffsetSizeAndStrideOpInterface >, InterfaceMethod< /*desc=*/[{ - Return a vector of all the static or dynamic sizes of the op. + Return a vector of all the static or dynamic offsets of the op. }], /*retTy=*/"::llvm::SmallVector<::mlir::OpFoldResult, 4>", /*methodName=*/"getMixedOffsets", -- GitLab From 6f7160eedb2db02f37d4ffd52fff7b0cf88b3fdc Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Tue, 16 Apr 2024 14:55:41 -0400 Subject: [PATCH 078/862] [SLP]Attempt to vectorize long stores, if short one failed. We can try to vectorize long store sequences, if short ones were unsuccessful because of the non-profitable vectorization. It should not increase compile time significantly (stores are sorted already, complexity is n x log n), but vectorize extra code. Metric: size..text Program size..text results results0 diff test-suite :: External/SPEC/CINT2006/400.perlbench/400.perlbench.test 1088012.00 1088236.00 0.0% test-suite :: SingleSource/UnitTests/matrix-types-spec.test 480396.00 480476.00 0.0% test-suite :: External/SPEC/CINT2017rate/525.x264_r/525.x264_r.test 664613.00 664661.00 0.0% test-suite :: External/SPEC/CINT2017speed/625.x264_s/625.x264_s.test 664613.00 664661.00 0.0% test-suite :: External/SPEC/CFP2017rate/510.parest_r/510.parest_r.test 2041105.00 2040961.00 -0.0% test-suite :: MultiSource/Applications/JM/lencod/lencod.test 836563.00 836387.00 -0.0% test-suite :: MultiSource/Benchmarks/7zip/7zip-benchmark.test 1035100.00 1032140.00 -0.3% In all benchmarks extra code gets vectorized Reviewers: RKSimon Reviewed By: RKSimon Pull Request: https://github.com/llvm/llvm-project/pull/88563 --- .../Transforms/Vectorize/SLPVectorizer.cpp | 106 ++++++++++++------ .../Transforms/SLPVectorizer/X86/pr46983.ll | 46 ++------ 2 files changed, 80 insertions(+), 72 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 7694627c3b04..806e8085038b 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -15164,10 +15164,6 @@ bool SLPVectorizerPass::vectorizeStores(ArrayRef Stores, BoUpSLP::ValueSet VectorizedStores; bool Changed = false; - // Stores the pair of stores (first_store, last_store) in a range, that were - // already tried to be vectorized. Allows to skip the store ranges that were - // already tried to be vectorized but the attempts were unsuccessful. - DenseSet> TriedSequences; struct StoreDistCompare { bool operator()(const std::pair &Op1, const std::pair &Op2) const { @@ -15209,8 +15205,10 @@ bool SLPVectorizerPass::vectorizeStores(ArrayRef Stores, Type *ValueTy = StoreTy; if (auto *Trunc = dyn_cast(Store->getValueOperand())) ValueTy = Trunc->getSrcTy(); - unsigned MinVF = PowerOf2Ceil(TTI->getStoreMinimumVF( - R.getMinVF(DL->getTypeStoreSizeInBits(StoreTy)), StoreTy, ValueTy)); + unsigned MinVF = std::max( + 2, PowerOf2Ceil(TTI->getStoreMinimumVF( + R.getMinVF(DL->getTypeStoreSizeInBits(StoreTy)), StoreTy, + ValueTy))); if (MaxVF < MinVF) { LLVM_DEBUG(dbgs() << "SLP: Vectorization infeasible as MaxVF (" << MaxVF @@ -15236,40 +15234,74 @@ bool SLPVectorizerPass::vectorizeStores(ArrayRef Stores, VF = Size > MaxVF ? NonPowerOf2VF : Size; Size *= 2; }); - unsigned StartIdx = 0; - for (unsigned Size : CandidateVFs) { - for (unsigned Cnt = StartIdx, E = Operands.size(); Cnt + Size <= E;) { - ArrayRef Slice = ArrayRef(Operands).slice(Cnt, Size); - assert( - all_of( - Slice, - [&](Value *V) { - return cast(V)->getValueOperand()->getType() == - cast(Slice.front()) - ->getValueOperand() - ->getType(); - }) && - "Expected all operands of same type."); - if (!VectorizedStores.count(Slice.front()) && - !VectorizedStores.count(Slice.back()) && - TriedSequences.insert(std::make_pair(Slice.front(), Slice.back())) - .second && - vectorizeStoreChain(Slice, R, Cnt, MinVF)) { - // Mark the vectorized stores so that we don't vectorize them again. - VectorizedStores.insert(Slice.begin(), Slice.end()); - Changed = true; - // If we vectorized initial block, no need to try to vectorize it - // again. - if (Cnt == StartIdx) - StartIdx += Size; - Cnt += Size; - continue; + unsigned End = Operands.size(); + unsigned Repeat = 0; + constexpr unsigned MaxAttempts = 2; + SmallBitVector Range(Operands.size()); + while (true) { + ++Repeat; + for (unsigned Size : CandidateVFs) { + int StartIdx = Range.find_first_unset(); + while (StartIdx != -1) { + int EndIdx = Range.find_next(StartIdx); + unsigned Sz = EndIdx == -1 ? End : EndIdx; + for (unsigned Cnt = StartIdx; Cnt + Size <= Sz;) { + ArrayRef Slice = ArrayRef(Operands).slice(Cnt, Size); + assert(all_of(Slice, + [&](Value *V) { + return cast(V) + ->getValueOperand() + ->getType() == + cast(Slice.front()) + ->getValueOperand() + ->getType(); + }) && + "Expected all operands of same type."); + if (vectorizeStoreChain(Slice, R, Cnt, MinVF)) { + // Mark the vectorized stores so that we don't vectorize them + // again. + VectorizedStores.insert(Slice.begin(), Slice.end()); + // Mark the vectorized stores so that we don't vectorize them + // again. + Changed = true; + // If we vectorized initial block, no need to try to vectorize + // it again. + Range.set(Cnt, Cnt + Size); + if (Cnt < StartIdx + MinVF) + Range.set(StartIdx, Cnt); + if (Cnt > EndIdx - Size - MinVF) { + Range.set(Cnt + Size, EndIdx); + End = Cnt; + } + Cnt += Size; + continue; + } + ++Cnt; + } + if (Sz >= End) + break; + StartIdx = Range.find_next_unset(EndIdx); } - ++Cnt; } - // Check if the whole array was vectorized already - exit. - if (StartIdx >= Operands.size()) + // All values vectorize - exit. + if (Range.all()) + break; + // Check if tried all attempts or no need for the last attempts at all. + if (Repeat >= MaxAttempts) + break; + constexpr unsigned MaxVFScale = 4; + constexpr unsigned StoresLimit = 16; + const unsigned MaxTotalNum = std::min( + std::max(StoresLimit, MaxVFScale * MaxVF), + bit_floor(static_cast(Range.find_last_unset() - + Range.find_first_unset() + 1))); + if (MaxVF >= MaxTotalNum) break; + // Last attempt to vectorize max number of elements, if all previous + // attempts were unsuccessful because of the cost issues. + CandidateVFs.clear(); + for (unsigned Size = MaxTotalNum; Size > MaxVF; Size /= 2) + CandidateVFs.push_back(Size); } } }; diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll index 75505f632a43..3deab0975ce7 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll @@ -100,41 +100,17 @@ define void @store_i8(ptr nocapture %0, i32 %1, i32 %2) { define void @store_i64(ptr nocapture %0, i32 %1, i32 %2) { ; SSE-LABEL: @store_i64( ; SSE-NEXT: [[TMP4:%.*]] = zext i32 [[TMP1:%.*]] to i64 -; SSE-NEXT: [[TMP5:%.*]] = load i64, ptr [[TMP0:%.*]], align 8, !tbaa [[TBAA5:![0-9]+]] -; SSE-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], [[TMP4]] -; SSE-NEXT: [[TMP7:%.*]] = lshr i64 [[TMP6]], 15 -; SSE-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP7]] to i32 -; SSE-NEXT: [[TMP9:%.*]] = icmp ult i32 [[TMP8]], 255 -; SSE-NEXT: [[TMP10:%.*]] = and i64 [[TMP7]], 4294967295 -; SSE-NEXT: [[TMP11:%.*]] = select i1 [[TMP9]], i64 [[TMP10]], i64 255 -; SSE-NEXT: store i64 [[TMP11]], ptr [[TMP0]], align 8, !tbaa [[TBAA5]] -; SSE-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 8 -; SSE-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP12]], align 8, !tbaa [[TBAA5]] -; SSE-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], [[TMP4]] -; SSE-NEXT: [[TMP15:%.*]] = lshr i64 [[TMP14]], 15 -; SSE-NEXT: [[TMP16:%.*]] = trunc i64 [[TMP15]] to i32 -; SSE-NEXT: [[TMP17:%.*]] = icmp ult i32 [[TMP16]], 255 -; SSE-NEXT: [[TMP18:%.*]] = and i64 [[TMP15]], 4294967295 -; SSE-NEXT: [[TMP19:%.*]] = select i1 [[TMP17]], i64 [[TMP18]], i64 255 -; SSE-NEXT: store i64 [[TMP19]], ptr [[TMP12]], align 8, !tbaa [[TBAA5]] -; SSE-NEXT: [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 16 -; SSE-NEXT: [[TMP21:%.*]] = load i64, ptr [[TMP20]], align 8, !tbaa [[TBAA5]] -; SSE-NEXT: [[TMP22:%.*]] = mul i64 [[TMP21]], [[TMP4]] -; SSE-NEXT: [[TMP23:%.*]] = lshr i64 [[TMP22]], 15 -; SSE-NEXT: [[TMP24:%.*]] = trunc i64 [[TMP23]] to i32 -; SSE-NEXT: [[TMP25:%.*]] = icmp ult i32 [[TMP24]], 255 -; SSE-NEXT: [[TMP26:%.*]] = and i64 [[TMP23]], 4294967295 -; SSE-NEXT: [[TMP27:%.*]] = select i1 [[TMP25]], i64 [[TMP26]], i64 255 -; SSE-NEXT: store i64 [[TMP27]], ptr [[TMP20]], align 8, !tbaa [[TBAA5]] -; SSE-NEXT: [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 24 -; SSE-NEXT: [[TMP29:%.*]] = load i64, ptr [[TMP28]], align 8, !tbaa [[TBAA5]] -; SSE-NEXT: [[TMP30:%.*]] = mul i64 [[TMP29]], [[TMP4]] -; SSE-NEXT: [[TMP31:%.*]] = lshr i64 [[TMP30]], 15 -; SSE-NEXT: [[TMP32:%.*]] = trunc i64 [[TMP31]] to i32 -; SSE-NEXT: [[TMP33:%.*]] = icmp ult i32 [[TMP32]], 255 -; SSE-NEXT: [[TMP34:%.*]] = and i64 [[TMP31]], 4294967295 -; SSE-NEXT: [[TMP35:%.*]] = select i1 [[TMP33]], i64 [[TMP34]], i64 255 -; SSE-NEXT: store i64 [[TMP35]], ptr [[TMP28]], align 8, !tbaa [[TBAA5]] +; SSE-NEXT: [[TMP5:%.*]] = load <4 x i64>, ptr [[TMP0:%.*]], align 8, !tbaa [[TBAA5:![0-9]+]] +; SSE-NEXT: [[TMP6:%.*]] = insertelement <4 x i64> poison, i64 [[TMP4]], i64 0 +; SSE-NEXT: [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP6]], <4 x i64> poison, <4 x i32> zeroinitializer +; SSE-NEXT: [[TMP8:%.*]] = mul <4 x i64> [[TMP5]], [[TMP7]] +; SSE-NEXT: [[TMP9:%.*]] = lshr <4 x i64> [[TMP8]], +; SSE-NEXT: [[TMP10:%.*]] = trunc <4 x i64> [[TMP9]] to <4 x i32> +; SSE-NEXT: [[TMP11:%.*]] = icmp ult <4 x i32> [[TMP10]], +; SSE-NEXT: [[TMP12:%.*]] = trunc <4 x i64> [[TMP9]] to <4 x i32> +; SSE-NEXT: [[TMP13:%.*]] = select <4 x i1> [[TMP11]], <4 x i32> [[TMP12]], <4 x i32> +; SSE-NEXT: [[TMP14:%.*]] = zext <4 x i32> [[TMP13]] to <4 x i64> +; SSE-NEXT: store <4 x i64> [[TMP14]], ptr [[TMP0]], align 8, !tbaa [[TBAA5]] ; SSE-NEXT: ret void ; ; AVX-LABEL: @store_i64( -- GitLab From eefee382186005d3662958e076c8e61e286ea1ab Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Wed, 17 Apr 2024 10:34:23 -0700 Subject: [PATCH 079/862] [libc] set cmake dependencies for condattr test (#89103) The entrypoints are not yet exposed on non-x86. Express this dependency to unbreak post submit. Fixes #88987 --- libc/src/pthread/pthread_condattr_destroy.cpp | 3 ++- libc/test/src/pthread/CMakeLists.txt | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/libc/src/pthread/pthread_condattr_destroy.cpp b/libc/src/pthread/pthread_condattr_destroy.cpp index 45cc011a4a92..41994c6941ff 100644 --- a/libc/src/pthread/pthread_condattr_destroy.cpp +++ b/libc/src/pthread/pthread_condattr_destroy.cpp @@ -14,7 +14,8 @@ namespace LIBC_NAMESPACE { -LLVM_LIBC_FUNCTION(int, pthread_condattr_destroy, (pthread_condattr_t * attr)) { +LLVM_LIBC_FUNCTION(int, pthread_condattr_destroy, + (pthread_condattr_t * attr [[gnu::unused]])) { // Initializing a pthread_condattr_t acquires no resources, so this is a // no-op. return 0; diff --git a/libc/test/src/pthread/CMakeLists.txt b/libc/test/src/pthread/CMakeLists.txt index 46f38422a537..51954a5babd2 100644 --- a/libc/test/src/pthread/CMakeLists.txt +++ b/libc/test/src/pthread/CMakeLists.txt @@ -50,4 +50,10 @@ add_libc_unittest( libc.include.errno libc.include.pthread libc.include.time + libc.src.pthread.pthread_condattr_destroy + libc.src.pthread.pthread_condattr_getclock + libc.src.pthread.pthread_condattr_getpshared + libc.src.pthread.pthread_condattr_init + libc.src.pthread.pthread_condattr_setclock + libc.src.pthread.pthread_condattr_setpshared ) -- GitLab From 825536039d667eeb933c590fe40c358fdea03a8d Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Wed, 17 Apr 2024 16:38:31 +0100 Subject: [PATCH 080/862] [CostModel][X86] Add basic GFNI target test coverage for shift/rotate costs --- .../Analysis/CostModel/X86/fshl-codesize.ll | 181 +++++++++++++ .../Analysis/CostModel/X86/fshl-latency.ll | 181 +++++++++++++ .../CostModel/X86/fshl-sizelatency.ll | 241 ++++++++++++++++++ llvm/test/Analysis/CostModel/X86/fshl.ll | 181 +++++++++++++ .../Analysis/CostModel/X86/fshr-codesize.ll | 181 +++++++++++++ .../Analysis/CostModel/X86/fshr-latency.ll | 181 +++++++++++++ .../CostModel/X86/fshr-sizelatency.ll | 241 ++++++++++++++++++ llvm/test/Analysis/CostModel/X86/fshr.ll | 181 +++++++++++++ .../CostModel/X86/vshift-ashr-codesize.ll | 85 ++++++ .../X86/vshift-ashr-cost-inseltpoison.ll | 87 +++++++ .../CostModel/X86/vshift-ashr-cost.ll | 87 +++++++ .../CostModel/X86/vshift-ashr-latency.ll | 93 +++++++ .../CostModel/X86/vshift-ashr-sizelatency.ll | 93 +++++++ .../CostModel/X86/vshift-lshr-codesize.ll | 65 +++++ .../X86/vshift-lshr-cost-inseltpoison.ll | 77 ++++++ .../CostModel/X86/vshift-lshr-cost.ll | 77 ++++++ .../CostModel/X86/vshift-lshr-latency.ll | 75 ++++++ .../CostModel/X86/vshift-lshr-sizelatency.ll | 77 ++++++ .../CostModel/X86/vshift-shl-codesize.ll | 57 +++++ .../X86/vshift-shl-cost-inseltpoison.ll | 81 ++++++ .../Analysis/CostModel/X86/vshift-shl-cost.ll | 81 ++++++ .../CostModel/X86/vshift-shl-latency.ll | 61 +++++ .../CostModel/X86/vshift-shl-sizelatency.ll | 73 ++++++ 23 files changed, 2737 insertions(+) diff --git a/llvm/test/Analysis/CostModel/X86/fshl-codesize.ll b/llvm/test/Analysis/CostModel/X86/fshl-codesize.ll index c212ecfc3eb9..c46e32ffb4ad 100644 --- a/llvm/test/Analysis/CostModel/X86/fshl-codesize.ll +++ b/llvm/test/Analysis/CostModel/X86/fshl-codesize.ll @@ -12,6 +12,7 @@ ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=code-size -mtriple=x86_64-apple-macosx10.8.0 -mcpu=goldmont | FileCheck %s --check-prefixes=GLM ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=code-size -mtriple=x86_64-apple-macosx10.8.0 -mcpu=bdver2 | FileCheck %s --check-prefixes=XOP ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=code-size -mtriple=x86_64-apple-macosx10.8.0 -mcpu=btver2 | FileCheck %s --check-prefixes=AVX1 +; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=code-size -mtriple=x86_64-apple-macosx10.8.0 -mcpu=tigerlake | FileCheck %s --check-prefixes=AVX512,AVX512GFNI target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.8.0" @@ -97,6 +98,13 @@ define void @var_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 %c64) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 %c64) %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128) @@ -182,6 +190,13 @@ define void @var_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i3 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 %c32) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 %c32) %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128) @@ -267,6 +282,13 @@ define void @var_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i ; XOP-NEXT: Cost Model: Found an estimated cost of 25 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 50 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 %c16) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 %c16) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128) @@ -352,6 +374,13 @@ define void @var_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> % ; XOP-NEXT: Cost Model: Found an estimated cost of 25 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 50 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 %c8) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 %c8) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128) @@ -463,6 +492,15 @@ define void @splatvar_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 21 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 42 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <8 x i64> %c512, <8 x i64> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer @@ -572,6 +610,15 @@ define void @splatvar_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <16 x i32> %c512, <16 x i32> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer @@ -681,6 +728,15 @@ define void @splatvar_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 25 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 50 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer @@ -790,6 +846,15 @@ define void @splatvar_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 25 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 50 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer @@ -881,6 +946,13 @@ define void @constant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7) %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) @@ -966,6 +1038,13 @@ define void @constant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 7) %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) @@ -1051,6 +1130,13 @@ define void @constant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 7) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) @@ -1136,6 +1222,13 @@ define void @constant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 27 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 7) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) @@ -1225,6 +1318,13 @@ define void @splatconstant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7) %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) @@ -1310,6 +1410,13 @@ define void @splatconstant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 5) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 5) %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) @@ -1395,6 +1502,13 @@ define void @splatconstant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; XOP-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 3) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) @@ -1480,6 +1594,13 @@ define void @splatconstant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; XOP-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 3) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) @@ -1690,6 +1811,13 @@ define void @var_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i ; XOP-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 %c16) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 %c16) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %c128) @@ -1775,6 +1903,13 @@ define void @var_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> % ; XOP-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 %c8) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 %c8) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %c128) @@ -2023,6 +2158,15 @@ define void @splatvar_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer @@ -2123,6 +2267,15 @@ define void @splatvar_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer @@ -2335,6 +2488,13 @@ define void @constant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 7) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) @@ -2420,6 +2580,13 @@ define void @constant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 7) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) @@ -2616,6 +2783,13 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; XOP-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 3) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) @@ -2694,6 +2868,13 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; XOP-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) diff --git a/llvm/test/Analysis/CostModel/X86/fshl-latency.ll b/llvm/test/Analysis/CostModel/X86/fshl-latency.ll index 487adfe79d94..fa32497c63ec 100644 --- a/llvm/test/Analysis/CostModel/X86/fshl-latency.ll +++ b/llvm/test/Analysis/CostModel/X86/fshl-latency.ll @@ -12,6 +12,7 @@ ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=latency -mtriple=x86_64-apple-macosx10.8.0 -mcpu=goldmont | FileCheck %s --check-prefixes=GLM ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=latency -mtriple=x86_64-apple-macosx10.8.0 -mcpu=bdver2 | FileCheck %s --check-prefixes=XOP ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=latency -mtriple=x86_64-apple-macosx10.8.0 -mcpu=btver2 | FileCheck %s --check-prefixes=AVX1 +; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=latency -mtriple=x86_64-apple-macosx10.8.0 -mcpu=tigerlake | FileCheck %s --check-prefixes=AVX512,AVX512GFNI target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.8.0" @@ -97,6 +98,13 @@ define void @var_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 %c64) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 %c64) %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128) @@ -182,6 +190,13 @@ define void @var_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i3 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 %c32) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 %c32) %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128) @@ -267,6 +282,13 @@ define void @var_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 %c16) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 %c16) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128) @@ -352,6 +374,13 @@ define void @var_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> % ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 %c8) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 21 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 58 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 51 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 %c8) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128) @@ -463,6 +492,15 @@ define void @splatvar_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <8 x i64> %c512, <8 x i64> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer @@ -563,6 +601,15 @@ define void @splatvar_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <16 x i32> %c512, <16 x i32> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer @@ -663,6 +710,15 @@ define void @splatvar_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer @@ -763,6 +819,15 @@ define void @splatvar_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer @@ -854,6 +919,13 @@ define void @constant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7) %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) @@ -939,6 +1011,13 @@ define void @constant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 7) %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) @@ -1024,6 +1103,13 @@ define void @constant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 7) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) @@ -1109,6 +1195,13 @@ define void @constant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 57 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 50 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 7) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) @@ -1198,6 +1291,13 @@ define void @splatconstant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7) %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) @@ -1276,6 +1376,13 @@ define void @splatconstant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 5) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 5) %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) @@ -1354,6 +1461,13 @@ define void @splatconstant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 3) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) @@ -1432,6 +1546,13 @@ define void @splatconstant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; XOP-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 3) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) @@ -1642,6 +1763,13 @@ define void @var_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i ; XOP-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 %c16) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 %c16) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %c128) @@ -1727,6 +1855,13 @@ define void @var_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> % ; XOP-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 %c8) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 %c8) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %c128) @@ -1975,6 +2110,15 @@ define void @splatvar_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer @@ -2075,6 +2219,15 @@ define void @splatvar_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer @@ -2287,6 +2440,13 @@ define void @constant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 7) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) @@ -2372,6 +2532,13 @@ define void @constant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 7) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) @@ -2568,6 +2735,13 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; XOP-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 3) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) @@ -2646,6 +2820,13 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; XOP-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) diff --git a/llvm/test/Analysis/CostModel/X86/fshl-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/fshl-sizelatency.ll index 52ad7e13c84c..832a574a9b33 100644 --- a/llvm/test/Analysis/CostModel/X86/fshl-sizelatency.ll +++ b/llvm/test/Analysis/CostModel/X86/fshl-sizelatency.ll @@ -12,6 +12,7 @@ ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mtriple=x86_64-apple-macosx10.8.0 -mcpu=goldmont | FileCheck %s --check-prefixes=GLM ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mtriple=x86_64-apple-macosx10.8.0 -mcpu=bdver2 | FileCheck %s --check-prefixes=XOP ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mtriple=x86_64-apple-macosx10.8.0 -mcpu=btver2 | FileCheck %s --check-prefixes=AVX1 +; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mtriple=x86_64-apple-macosx10.8.0 -mcpu=tigerlake | FileCheck %s --check-prefixes=AVX512GFNI target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.8.0" @@ -97,6 +98,13 @@ define void @var_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64 ; XOP-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 %c64) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 %c64) %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128) @@ -182,6 +190,13 @@ define void @var_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i3 ; XOP-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 %c32) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 %c32) %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128) @@ -267,6 +282,13 @@ define void @var_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i ; XOP-NEXT: Cost Model: Found an estimated cost of 31 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 62 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 %c16) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 %c16) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128) @@ -352,6 +374,13 @@ define void @var_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> % ; XOP-NEXT: Cost Model: Found an estimated cost of 31 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 62 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 %c8) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 39 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 38 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 %c8) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128) @@ -463,6 +492,15 @@ define void @splatvar_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <8 x i64> %c512, <8 x i64> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer @@ -572,6 +610,15 @@ define void @splatvar_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <16 x i32> %c512, <16 x i32> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer @@ -681,6 +728,15 @@ define void @splatvar_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 66 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer @@ -790,6 +846,15 @@ define void @splatvar_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 31 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 62 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer @@ -881,6 +946,13 @@ define void @constant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7) %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) @@ -966,6 +1038,13 @@ define void @constant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 7) %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) @@ -1051,6 +1130,13 @@ define void @constant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 29 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 58 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 7) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) @@ -1136,6 +1222,13 @@ define void @constant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 29 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 58 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 38 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 37 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 7) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) @@ -1225,6 +1318,13 @@ define void @splatconstant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 26 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 52 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7) %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) @@ -1310,6 +1410,13 @@ define void @splatconstant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 26 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 52 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 5) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 5) %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) @@ -1395,6 +1502,13 @@ define void @splatconstant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; XOP-NEXT: Cost Model: Found an estimated cost of 27 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 54 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 3) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) @@ -1480,6 +1594,13 @@ define void @splatconstant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; XOP-NEXT: Cost Model: Found an estimated cost of 29 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 58 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 3) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) @@ -1562,6 +1683,13 @@ define void @var_rotate_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64 ; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %a64, i64 %c64) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %a64, i64 %c64) %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> %c128) @@ -1647,6 +1775,13 @@ define void @var_rotate_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i3 ; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %a32, i32 %c32) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %a32, i32 %c32) %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> %c128) @@ -1732,6 +1867,13 @@ define void @var_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i ; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 %c16) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 %c16) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %c128) @@ -1817,6 +1959,13 @@ define void @var_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> % ; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 %c8) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 %c8) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %c128) @@ -1919,6 +2068,15 @@ define void @splatvar_rotate_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <8 x i64> %c512, <8 x i64> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer @@ -2019,6 +2177,15 @@ define void @splatvar_rotate_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <16 x i32> %c512, <16 x i32> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer @@ -2119,6 +2286,15 @@ define void @splatvar_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer @@ -2219,6 +2395,15 @@ define void @splatvar_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer @@ -2303,6 +2488,13 @@ define void @constant_rotate_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %a64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %a64, i64 7) %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> ) @@ -2388,6 +2580,13 @@ define void @constant_rotate_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %a32, i32 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %a32, i32 7) %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> ) @@ -2473,6 +2672,13 @@ define void @constant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 7) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) @@ -2558,6 +2764,13 @@ define void @constant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 7) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) @@ -2640,6 +2853,13 @@ define void @splatconstant_rotate_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %a64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %a64, i64 7) %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> ) @@ -2718,6 +2938,13 @@ define void @splatconstant_rotate_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %a32, i32 5) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %a32, i32 5) %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> ) @@ -2796,6 +3023,13 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 3) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) @@ -2874,6 +3108,13 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) diff --git a/llvm/test/Analysis/CostModel/X86/fshl.ll b/llvm/test/Analysis/CostModel/X86/fshl.ll index 4e688c29c7ea..311d8d5ed7d2 100644 --- a/llvm/test/Analysis/CostModel/X86/fshl.ll +++ b/llvm/test/Analysis/CostModel/X86/fshl.ll @@ -12,6 +12,7 @@ ; RUN: opt < %s -passes="print" 2>&1 -disable-output -mtriple=x86_64-apple-macosx10.8.0 -mcpu=goldmont | FileCheck %s --check-prefixes=GLM ; RUN: opt < %s -passes="print" 2>&1 -disable-output -mtriple=x86_64-apple-macosx10.8.0 -mcpu=bdver2 | FileCheck %s --check-prefixes=XOP ; RUN: opt < %s -passes="print" 2>&1 -disable-output -mtriple=x86_64-apple-macosx10.8.0 -mcpu=btver2 | FileCheck %s --check-prefixes=AVX1 +; RUN: opt < %s -passes="print" 2>&1 -disable-output -mtriple=x86_64-apple-macosx10.8.0 -mcpu=tigerlake | FileCheck %s --check-prefixes=AVX512,AVX512GFNI target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.8.0" @@ -97,6 +98,13 @@ define void @var_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 %c64) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 %c64) %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128) @@ -182,6 +190,13 @@ define void @var_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i3 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 %c32) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 %c32) %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128) @@ -267,6 +282,13 @@ define void @var_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 %c16) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 %c16) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128) @@ -352,6 +374,13 @@ define void @var_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> % ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 %c8) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 %c8) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128) @@ -463,6 +492,15 @@ define void @splatvar_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 38 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <8 x i64> %c512, <8 x i64> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer @@ -563,6 +601,15 @@ define void @splatvar_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 38 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <16 x i32> %c512, <16 x i32> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer @@ -663,6 +710,15 @@ define void @splatvar_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 38 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer @@ -763,6 +819,15 @@ define void @splatvar_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer @@ -854,6 +919,13 @@ define void @constant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7) %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) @@ -932,6 +1004,13 @@ define void @constant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 7) %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) @@ -1017,6 +1096,13 @@ define void @constant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 7) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) @@ -1102,6 +1188,13 @@ define void @constant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 7) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) @@ -1191,6 +1284,13 @@ define void @splatconstant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7) %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) @@ -1269,6 +1369,13 @@ define void @splatconstant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 5) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 5) %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) @@ -1347,6 +1454,13 @@ define void @splatconstant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 3) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) @@ -1425,6 +1539,13 @@ define void @splatconstant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 3) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) @@ -1635,6 +1756,13 @@ define void @var_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i ; XOP-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 %c16) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 %c16) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %c128) @@ -1720,6 +1848,13 @@ define void @var_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> % ; XOP-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 %c8) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 %c8) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %c128) @@ -1968,6 +2103,15 @@ define void @splatvar_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer @@ -2068,6 +2212,15 @@ define void @splatvar_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer @@ -2273,6 +2426,13 @@ define void @constant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 7) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) @@ -2358,6 +2518,13 @@ define void @constant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 7) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) @@ -2554,6 +2721,13 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; XOP-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 3) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) @@ -2632,6 +2806,13 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; XOP-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) diff --git a/llvm/test/Analysis/CostModel/X86/fshr-codesize.ll b/llvm/test/Analysis/CostModel/X86/fshr-codesize.ll index 0e76246fad85..f9d30e4ced3e 100644 --- a/llvm/test/Analysis/CostModel/X86/fshr-codesize.ll +++ b/llvm/test/Analysis/CostModel/X86/fshr-codesize.ll @@ -12,6 +12,7 @@ ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=code-size -mtriple=x86_64-apple-macosx10.8.0 -mcpu=goldmont | FileCheck %s --check-prefixes=GLM ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=code-size -mtriple=x86_64-apple-macosx10.8.0 -mcpu=bdver2 | FileCheck %s --check-prefixes=XOP ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=code-size -mtriple=x86_64-apple-macosx10.8.0 -mcpu=btver2 | FileCheck %s --check-prefixes=AVX1 +; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=code-size -mtriple=x86_64-apple-macosx10.8.0 -mcpu=tigerlake | FileCheck %s --check-prefixes=AVX512,AVX512GFNI target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.8.0" @@ -97,6 +98,13 @@ define void @var_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 %c64) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 %c64) %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128) @@ -182,6 +190,13 @@ define void @var_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i3 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 %c32) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 %c32) %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128) @@ -267,6 +282,13 @@ define void @var_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i ; XOP-NEXT: Cost Model: Found an estimated cost of 25 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 50 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 %c16) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 %c16) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128) @@ -352,6 +374,13 @@ define void @var_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> % ; XOP-NEXT: Cost Model: Found an estimated cost of 25 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 50 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 %c8) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 %c8) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128) @@ -463,6 +492,15 @@ define void @splatvar_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 21 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 42 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <8 x i64> %c512, <8 x i64> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer @@ -572,6 +610,15 @@ define void @splatvar_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <16 x i32> %c512, <16 x i32> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer @@ -681,6 +728,15 @@ define void @splatvar_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 25 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 50 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer @@ -790,6 +846,15 @@ define void @splatvar_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 25 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 50 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer @@ -881,6 +946,13 @@ define void @constant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7) %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) @@ -966,6 +1038,13 @@ define void @constant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 7) %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) @@ -1051,6 +1130,13 @@ define void @constant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 7) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) @@ -1136,6 +1222,13 @@ define void @constant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 27 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 7) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) @@ -1225,6 +1318,13 @@ define void @splatconstant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7) %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) @@ -1310,6 +1410,13 @@ define void @splatconstant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 5) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 5) %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) @@ -1395,6 +1502,13 @@ define void @splatconstant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; XOP-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 3) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) @@ -1480,6 +1594,13 @@ define void @splatconstant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; XOP-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 3) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) @@ -1690,6 +1811,13 @@ define void @var_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 %c16) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 %c16) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %c128) @@ -1775,6 +1903,13 @@ define void @var_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> % ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 %c8) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 %c8) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %c128) @@ -2023,6 +2158,15 @@ define void @splatvar_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer @@ -2123,6 +2267,15 @@ define void @splatvar_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer @@ -2335,6 +2488,13 @@ define void @constant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 7) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) @@ -2420,6 +2580,13 @@ define void @constant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 7) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) @@ -2616,6 +2783,13 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 3) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) @@ -2694,6 +2868,13 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) diff --git a/llvm/test/Analysis/CostModel/X86/fshr-latency.ll b/llvm/test/Analysis/CostModel/X86/fshr-latency.ll index 73e32dc92a69..ed2227591847 100644 --- a/llvm/test/Analysis/CostModel/X86/fshr-latency.ll +++ b/llvm/test/Analysis/CostModel/X86/fshr-latency.ll @@ -12,6 +12,7 @@ ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=latency -mtriple=x86_64-apple-macosx10.8.0 -mcpu=goldmont | FileCheck %s --check-prefixes=GLM ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=latency -mtriple=x86_64-apple-macosx10.8.0 -mcpu=bdver2 | FileCheck %s --check-prefixes=XOP ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=latency -mtriple=x86_64-apple-macosx10.8.0 -mcpu=btver2 | FileCheck %s --check-prefixes=AVX1 +; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=latency -mtriple=x86_64-apple-macosx10.8.0 -mcpu=tigerlake | FileCheck %s --check-prefixes=AVX512,AVX512GFNI target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.8.0" @@ -97,6 +98,13 @@ define void @var_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 %c64) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 %c64) %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128) @@ -182,6 +190,13 @@ define void @var_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i3 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 %c32) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 %c32) %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128) @@ -267,6 +282,13 @@ define void @var_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 %c16) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 %c16) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128) @@ -352,6 +374,13 @@ define void @var_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> % ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 %c8) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 21 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 58 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 51 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 %c8) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128) @@ -463,6 +492,15 @@ define void @splatvar_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <8 x i64> %c512, <8 x i64> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer @@ -563,6 +601,15 @@ define void @splatvar_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <16 x i32> %c512, <16 x i32> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer @@ -663,6 +710,15 @@ define void @splatvar_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer @@ -763,6 +819,15 @@ define void @splatvar_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer @@ -854,6 +919,13 @@ define void @constant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7) %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) @@ -939,6 +1011,13 @@ define void @constant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 7) %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) @@ -1024,6 +1103,13 @@ define void @constant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 7) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) @@ -1109,6 +1195,13 @@ define void @constant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 57 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 50 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 7) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) @@ -1198,6 +1291,13 @@ define void @splatconstant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7) %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) @@ -1276,6 +1376,13 @@ define void @splatconstant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 5) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 5) %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) @@ -1354,6 +1461,13 @@ define void @splatconstant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 3) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) @@ -1432,6 +1546,13 @@ define void @splatconstant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; XOP-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 3) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) @@ -1642,6 +1763,13 @@ define void @var_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i ; XOP-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 %c16) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 %c16) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %c128) @@ -1727,6 +1855,13 @@ define void @var_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> % ; XOP-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 %c8) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 %c8) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %c128) @@ -1975,6 +2110,15 @@ define void @splatvar_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer @@ -2075,6 +2219,15 @@ define void @splatvar_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer @@ -2287,6 +2440,13 @@ define void @constant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 7) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) @@ -2372,6 +2532,13 @@ define void @constant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 7) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) @@ -2568,6 +2735,13 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; XOP-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 3) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) @@ -2646,6 +2820,13 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; XOP-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) diff --git a/llvm/test/Analysis/CostModel/X86/fshr-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/fshr-sizelatency.ll index 2d106c6dd306..8931781f70bd 100644 --- a/llvm/test/Analysis/CostModel/X86/fshr-sizelatency.ll +++ b/llvm/test/Analysis/CostModel/X86/fshr-sizelatency.ll @@ -12,6 +12,7 @@ ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mtriple=x86_64-apple-macosx10.8.0 -mcpu=goldmont | FileCheck %s --check-prefixes=GLM ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mtriple=x86_64-apple-macosx10.8.0 -mcpu=bdver2 | FileCheck %s --check-prefixes=XOP ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mtriple=x86_64-apple-macosx10.8.0 -mcpu=btver2 | FileCheck %s --check-prefixes=AVX1 +; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mtriple=x86_64-apple-macosx10.8.0 -mcpu=tigerlake | FileCheck %s --check-prefixes=AVX512GFNI target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.8.0" @@ -97,6 +98,13 @@ define void @var_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64 ; XOP-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 %c64) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 %c64) %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128) @@ -182,6 +190,13 @@ define void @var_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i3 ; XOP-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 %c32) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 %c32) %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128) @@ -267,6 +282,13 @@ define void @var_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i ; XOP-NEXT: Cost Model: Found an estimated cost of 31 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 62 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 %c16) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 %c16) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128) @@ -352,6 +374,13 @@ define void @var_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> % ; XOP-NEXT: Cost Model: Found an estimated cost of 31 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 62 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 %c8) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 39 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 38 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 %c8) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128) @@ -463,6 +492,15 @@ define void @splatvar_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <8 x i64> %c512, <8 x i64> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer @@ -572,6 +610,15 @@ define void @splatvar_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <16 x i32> %c512, <16 x i32> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer @@ -681,6 +728,15 @@ define void @splatvar_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 66 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer @@ -790,6 +846,15 @@ define void @splatvar_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 31 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 62 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer @@ -881,6 +946,13 @@ define void @constant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7) %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) @@ -966,6 +1038,13 @@ define void @constant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 7) %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) @@ -1051,6 +1130,13 @@ define void @constant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 29 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 58 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 7) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) @@ -1136,6 +1222,13 @@ define void @constant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 29 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 58 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 38 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 37 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 7) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) @@ -1225,6 +1318,13 @@ define void @splatconstant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 26 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 52 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7) %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) @@ -1310,6 +1410,13 @@ define void @splatconstant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 26 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 52 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 5) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 5) %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) @@ -1395,6 +1502,13 @@ define void @splatconstant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; XOP-NEXT: Cost Model: Found an estimated cost of 27 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 54 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 3) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) @@ -1480,6 +1594,13 @@ define void @splatconstant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; XOP-NEXT: Cost Model: Found an estimated cost of 29 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 58 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 3) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) @@ -1562,6 +1683,13 @@ define void @var_rotate_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64 ; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %a64, i64 %c64) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %a64, i64 %c64) %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> %c128) @@ -1647,6 +1775,13 @@ define void @var_rotate_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i3 ; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %a32, i32 %c32) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %a32, i32 %c32) %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> %c128) @@ -1732,6 +1867,13 @@ define void @var_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i ; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 %c16) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 %c16) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %c128) @@ -1817,6 +1959,13 @@ define void @var_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> % ; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 %c8) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 %c8) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %c128) @@ -1919,6 +2068,15 @@ define void @splatvar_rotate_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <8 x i64> %c512, <8 x i64> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer @@ -2019,6 +2177,15 @@ define void @splatvar_rotate_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <16 x i32> %c512, <16 x i32> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer @@ -2119,6 +2286,15 @@ define void @splatvar_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer @@ -2219,6 +2395,15 @@ define void @splatvar_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer @@ -2303,6 +2488,13 @@ define void @constant_rotate_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %a64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %a64, i64 7) %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> ) @@ -2388,6 +2580,13 @@ define void @constant_rotate_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %a32, i32 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %a32, i32 7) %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> ) @@ -2473,6 +2672,13 @@ define void @constant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 7) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) @@ -2558,6 +2764,13 @@ define void @constant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 7) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) @@ -2640,6 +2853,13 @@ define void @splatconstant_rotate_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %a64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %a64, i64 7) %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> ) @@ -2718,6 +2938,13 @@ define void @splatconstant_rotate_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %a32, i32 5) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %a32, i32 5) %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> ) @@ -2796,6 +3023,13 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 3) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) @@ -2874,6 +3108,13 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) diff --git a/llvm/test/Analysis/CostModel/X86/fshr.ll b/llvm/test/Analysis/CostModel/X86/fshr.ll index 9565630677d5..ca9ddcc52938 100644 --- a/llvm/test/Analysis/CostModel/X86/fshr.ll +++ b/llvm/test/Analysis/CostModel/X86/fshr.ll @@ -12,6 +12,7 @@ ; RUN: opt < %s -passes="print" 2>&1 -disable-output -mtriple=x86_64-apple-macosx10.8.0 -mcpu=goldmont | FileCheck %s --check-prefixes=GLM ; RUN: opt < %s -passes="print" 2>&1 -disable-output -mtriple=x86_64-apple-macosx10.8.0 -mcpu=bdver2 | FileCheck %s --check-prefixes=XOP ; RUN: opt < %s -passes="print" 2>&1 -disable-output -mtriple=x86_64-apple-macosx10.8.0 -mcpu=btver2 | FileCheck %s --check-prefixes=AVX1 +; RUN: opt < %s -passes="print" 2>&1 -disable-output -mtriple=x86_64-apple-macosx10.8.0 -mcpu=tigerlake | FileCheck %s --check-prefixes=AVX512,AVX512GFNI target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.8.0" @@ -97,6 +98,13 @@ define void @var_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 %c64) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 %c64) %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128) @@ -182,6 +190,13 @@ define void @var_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i3 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 %c32) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 %c32) %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128) @@ -267,6 +282,13 @@ define void @var_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 %c16) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 %c16) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128) @@ -352,6 +374,13 @@ define void @var_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> % ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 %c8) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 %c8) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128) @@ -463,6 +492,15 @@ define void @splatvar_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 38 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <8 x i64> %c512, <8 x i64> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer @@ -563,6 +601,15 @@ define void @splatvar_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 38 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <16 x i32> %c512, <16 x i32> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer @@ -663,6 +710,15 @@ define void @splatvar_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 38 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer @@ -763,6 +819,15 @@ define void @splatvar_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer @@ -854,6 +919,13 @@ define void @constant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7) %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) @@ -932,6 +1004,13 @@ define void @constant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 7) %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) @@ -1017,6 +1096,13 @@ define void @constant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 7) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) @@ -1102,6 +1188,13 @@ define void @constant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 7) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) @@ -1191,6 +1284,13 @@ define void @splatconstant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7) %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) @@ -1269,6 +1369,13 @@ define void @splatconstant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 5) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 5) %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) @@ -1347,6 +1454,13 @@ define void @splatconstant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 3) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) @@ -1425,6 +1539,13 @@ define void @splatconstant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 3) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) @@ -1635,6 +1756,13 @@ define void @var_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i ; XOP-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 %c16) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 %c16) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %c128) @@ -1720,6 +1848,13 @@ define void @var_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> % ; XOP-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 %c8) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 %c8) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %c128) @@ -1968,6 +2103,15 @@ define void @splatvar_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer @@ -2068,6 +2212,15 @@ define void @splatvar_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer @@ -2273,6 +2426,13 @@ define void @constant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 7) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) @@ -2358,6 +2518,13 @@ define void @constant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 7) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) @@ -2554,6 +2721,13 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; XOP-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 3) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) @@ -2632,6 +2806,13 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; XOP-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) diff --git a/llvm/test/Analysis/CostModel/X86/vshift-ashr-codesize.ll b/llvm/test/Analysis/CostModel/X86/vshift-ashr-codesize.ll index 8e3f38c796e6..a3c24bdd1a88 100644 --- a/llvm/test/Analysis/CostModel/X86/vshift-ashr-codesize.ll +++ b/llvm/test/Analysis/CostModel/X86/vshift-ashr-codesize.ll @@ -15,6 +15,7 @@ ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mcpu=btver2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX1 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mcpu=tigerlake | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512GFNI ; Verify the cost of vector logical shift right instructions. @@ -232,6 +233,10 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v8i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift ; %shift = ashr <8 x i16> %a, %b ret <8 x i16> %shift @@ -265,6 +270,10 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = ashr <16 x i16> %a, %b ret <16 x i16> %shift @@ -298,6 +307,10 @@ define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = ashr <32 x i16> %a, %b ret <32 x i16> %shift @@ -327,6 +340,10 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <16 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <16 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = ashr <16 x i8> %a, %b ret <16 x i8> %shift @@ -389,6 +406,10 @@ define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %shift = ashr <64 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %shift = ashr <64 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = ashr <64 x i8> %a, %b ret <64 x i8> %shift @@ -778,6 +799,12 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, i16 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = ashr <32 x i16> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i16> undef, i16 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = ashr <32 x i16> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %insert = insertelement <32 x i16> undef, i16 %b, i32 0 %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer @@ -827,6 +854,12 @@ define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, i8 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <16 x i8> %insert, <16 x i8> undef, <16 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = ashr <16 x i8> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <16 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <16 x i8> %insert, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = ashr <16 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %insert = insertelement <16 x i8> undef, i8 %b, i32 0 %splat = shufflevector <16 x i8> %insert, <16 x i8> undef, <16 x i32> zeroinitializer @@ -882,6 +915,12 @@ define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, i8 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i8> %insert, <32 x i8> undef, <32 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %shift = ashr <32 x i8> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i8> %insert, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %shift = ashr <32 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift ; %insert = insertelement <32 x i8> undef, i8 %b, i32 0 %splat = shufflevector <32 x i8> %insert, <32 x i8> undef, <32 x i32> zeroinitializer @@ -937,6 +976,12 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, i8 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %shift = ashr <64 x i8> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <64 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %shift = ashr <64 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %insert = insertelement <64 x i8> undef, i8 %b, i32 0 %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer @@ -1158,6 +1203,10 @@ define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v8i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift ; %shift = ashr <8 x i16> %a, ret <8 x i16> %shift @@ -1191,6 +1240,10 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = ashr <16 x i16> %a, ret <16 x i16> %shift @@ -1224,6 +1277,10 @@ define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = ashr <32 x i16> %a, ret <32 x i16> %shift @@ -1253,6 +1310,10 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <16 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = ashr <16 x i8> %a, ret <16 x i8> %shift @@ -1315,6 +1376,10 @@ define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %shift = ashr <64 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %shift = ashr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = ashr <64 x i8> %a, ret <64 x i8> %shift @@ -1531,6 +1596,10 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = ashr <16 x i16> %a, ret <16 x i16> %shift @@ -1568,6 +1637,10 @@ define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = ashr <32 x i16> %a, ret <32 x i16> %shift @@ -1601,6 +1674,10 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <16 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = ashr <16 x i8> %a, ret <16 x i8> %shift @@ -1634,6 +1711,10 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v32i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <32 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift ; %shift = ashr <32 x i8> %a, ret <32 x i8> %shift @@ -1667,6 +1748,10 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <64 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = ashr <64 x i8> %a, ret <64 x i8> %shift diff --git a/llvm/test/Analysis/CostModel/X86/vshift-ashr-cost-inseltpoison.ll b/llvm/test/Analysis/CostModel/X86/vshift-ashr-cost-inseltpoison.ll index d4ece3b2a113..67679f2cb856 100644 --- a/llvm/test/Analysis/CostModel/X86/vshift-ashr-cost-inseltpoison.ll +++ b/llvm/test/Analysis/CostModel/X86/vshift-ashr-cost-inseltpoison.ll @@ -15,6 +15,7 @@ ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=btver2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX1 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=tigerlake | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512GFNI ; Verify the cost of vector arithmetic shift right instructions. @@ -228,6 +229,10 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; AVX512BWVL-LABEL: 'var_shift_v8i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift ; %shift = ashr <8 x i16> %a, %b ret <8 x i16> %shift @@ -269,6 +274,10 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; AVX512BWVL-LABEL: 'var_shift_v16i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = ashr <16 x i16> %a, %b ret <16 x i16> %shift @@ -310,6 +319,10 @@ define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) { ; AVX512BWVL-LABEL: 'var_shift_v32i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = ashr <32 x i16> %a, %b ret <32 x i16> %shift @@ -351,6 +364,10 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; AVX512BWVL-LABEL: 'var_shift_v16i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <16 x i8> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <16 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift ; %shift = ashr <16 x i8> %a, %b ret <16 x i8> %shift @@ -392,6 +409,10 @@ define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { ; AVX512BWVL-LABEL: 'var_shift_v32i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shift = ashr <32 x i8> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shift = ashr <32 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = ashr <32 x i8> %a, %b ret <32 x i8> %shift @@ -433,6 +454,10 @@ define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) { ; AVX512BWVL-LABEL: 'var_shift_v64i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = ashr <64 x i8> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = ashr <64 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = ashr <64 x i8> %a, %b ret <64 x i8> %shift @@ -834,6 +859,12 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, i16 %b) { ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> poison, <32 x i32> zeroinitializer ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = ashr <32 x i16> %a, %splat ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i16> poison, i16 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> poison, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = ashr <32 x i16> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %insert = insertelement <32 x i16> poison, i16 %b, i32 0 %splat = shufflevector <32 x i16> %insert, <32 x i16> poison, <32 x i32> zeroinitializer @@ -950,6 +981,12 @@ define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, i8 %b) { ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i8> %insert, <32 x i8> poison, <32 x i32> zeroinitializer ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = ashr <32 x i8> %a, %splat ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i8> poison, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i8> %insert, <32 x i8> poison, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = ashr <32 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %insert = insertelement <32 x i8> poison, i8 %b, i32 0 %splat = shufflevector <32 x i8> %insert, <32 x i8> poison, <32 x i32> zeroinitializer @@ -1017,6 +1054,12 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, i8 %b) { ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> poison, <64 x i32> zeroinitializer ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = ashr <64 x i8> %a, %splat ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <64 x i8> poison, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> poison, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = ashr <64 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %insert = insertelement <64 x i8> poison, i8 %b, i32 0 %splat = shufflevector <64 x i8> %insert, <64 x i8> poison, <64 x i32> zeroinitializer @@ -1226,6 +1269,10 @@ define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v8i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift ; %shift = ashr <8 x i16> %a, ret <8 x i16> %shift @@ -1267,6 +1314,10 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v16i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = ashr <16 x i16> %a, ret <16 x i16> %shift @@ -1308,6 +1359,10 @@ define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v32i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = ashr <32 x i16> %a, ret <32 x i16> %shift @@ -1349,6 +1404,10 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v16i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <16 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift ; %shift = ashr <16 x i8> %a, ret <16 x i8> %shift @@ -1390,6 +1449,10 @@ define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v32i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shift = ashr <32 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shift = ashr <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = ashr <32 x i8> %a, ret <32 x i8> %shift @@ -1431,6 +1494,10 @@ define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v64i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = ashr <64 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = ashr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = ashr <64 x i8> %a, ret <64 x i8> %shift @@ -1631,6 +1698,10 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v16i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = ashr <16 x i16> %a, ret <16 x i16> %shift @@ -1672,6 +1743,10 @@ define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v32i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = ashr <32 x i16> %a, ret <32 x i16> %shift @@ -1713,6 +1788,10 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v16i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift ; %shift = ashr <16 x i8> %a, ret <16 x i8> %shift @@ -1754,6 +1833,10 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v32i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = ashr <32 x i8> %a, ret <32 x i8> %shift @@ -1795,6 +1878,10 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v64i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <64 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = ashr <64 x i8> %a, ret <64 x i8> %shift diff --git a/llvm/test/Analysis/CostModel/X86/vshift-ashr-cost.ll b/llvm/test/Analysis/CostModel/X86/vshift-ashr-cost.ll index f8fb2d76a778..efd378ee0b5a 100644 --- a/llvm/test/Analysis/CostModel/X86/vshift-ashr-cost.ll +++ b/llvm/test/Analysis/CostModel/X86/vshift-ashr-cost.ll @@ -15,6 +15,7 @@ ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=btver2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX1 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=tigerlake | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512GFNI ; Verify the cost of vector arithmetic shift right instructions. @@ -228,6 +229,10 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; AVX512BWVL-LABEL: 'var_shift_v8i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift ; %shift = ashr <8 x i16> %a, %b ret <8 x i16> %shift @@ -269,6 +274,10 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; AVX512BWVL-LABEL: 'var_shift_v16i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = ashr <16 x i16> %a, %b ret <16 x i16> %shift @@ -310,6 +319,10 @@ define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) { ; AVX512BWVL-LABEL: 'var_shift_v32i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = ashr <32 x i16> %a, %b ret <32 x i16> %shift @@ -351,6 +364,10 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; AVX512BWVL-LABEL: 'var_shift_v16i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <16 x i8> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <16 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift ; %shift = ashr <16 x i8> %a, %b ret <16 x i8> %shift @@ -392,6 +409,10 @@ define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { ; AVX512BWVL-LABEL: 'var_shift_v32i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shift = ashr <32 x i8> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shift = ashr <32 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = ashr <32 x i8> %a, %b ret <32 x i8> %shift @@ -433,6 +454,10 @@ define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) { ; AVX512BWVL-LABEL: 'var_shift_v64i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = ashr <64 x i8> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = ashr <64 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = ashr <64 x i8> %a, %b ret <64 x i8> %shift @@ -834,6 +859,12 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, i16 %b) { ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = ashr <32 x i16> %a, %splat ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i16> undef, i16 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = ashr <32 x i16> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %insert = insertelement <32 x i16> undef, i16 %b, i32 0 %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer @@ -950,6 +981,12 @@ define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, i8 %b) { ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i8> %insert, <32 x i8> undef, <32 x i32> zeroinitializer ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = ashr <32 x i8> %a, %splat ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i8> %insert, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = ashr <32 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %insert = insertelement <32 x i8> undef, i8 %b, i32 0 %splat = shufflevector <32 x i8> %insert, <32 x i8> undef, <32 x i32> zeroinitializer @@ -1017,6 +1054,12 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, i8 %b) { ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = ashr <64 x i8> %a, %splat ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <64 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = ashr <64 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %insert = insertelement <64 x i8> undef, i8 %b, i32 0 %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer @@ -1226,6 +1269,10 @@ define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v8i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift ; %shift = ashr <8 x i16> %a, ret <8 x i16> %shift @@ -1267,6 +1314,10 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v16i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = ashr <16 x i16> %a, ret <16 x i16> %shift @@ -1308,6 +1359,10 @@ define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v32i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = ashr <32 x i16> %a, ret <32 x i16> %shift @@ -1349,6 +1404,10 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v16i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <16 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift ; %shift = ashr <16 x i8> %a, ret <16 x i8> %shift @@ -1390,6 +1449,10 @@ define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v32i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shift = ashr <32 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shift = ashr <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = ashr <32 x i8> %a, ret <32 x i8> %shift @@ -1431,6 +1494,10 @@ define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v64i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = ashr <64 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = ashr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = ashr <64 x i8> %a, ret <64 x i8> %shift @@ -1631,6 +1698,10 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v16i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = ashr <16 x i16> %a, ret <16 x i16> %shift @@ -1672,6 +1743,10 @@ define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v32i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = ashr <32 x i16> %a, ret <32 x i16> %shift @@ -1713,6 +1788,10 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v16i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift ; %shift = ashr <16 x i8> %a, ret <16 x i8> %shift @@ -1754,6 +1833,10 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v32i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = ashr <32 x i8> %a, ret <32 x i8> %shift @@ -1795,6 +1878,10 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v64i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <64 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = ashr <64 x i8> %a, ret <64 x i8> %shift diff --git a/llvm/test/Analysis/CostModel/X86/vshift-ashr-latency.ll b/llvm/test/Analysis/CostModel/X86/vshift-ashr-latency.ll index af5278e355cf..cd4189d4a7f8 100644 --- a/llvm/test/Analysis/CostModel/X86/vshift-ashr-latency.ll +++ b/llvm/test/Analysis/CostModel/X86/vshift-ashr-latency.ll @@ -15,6 +15,7 @@ ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mcpu=slm | FileCheck %s --check-prefixes=SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mcpu=goldmont | FileCheck %s --check-prefixes=SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mcpu=btver2 | FileCheck %s --check-prefixes=AVX,AVX1 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mcpu=tigerlake | FileCheck %s --check-prefixes=AVX512,AVX512GFNI ; Verify the cost of vector logical shift right instructions. @@ -232,6 +233,10 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v8i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift ; %shift = ashr <8 x i16> %a, %b ret <8 x i16> %shift @@ -265,6 +270,10 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = ashr <16 x i16> %a, %b ret <16 x i16> %shift @@ -298,6 +307,10 @@ define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = ashr <32 x i16> %a, %b ret <32 x i16> %shift @@ -331,6 +344,10 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = ashr <16 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = ashr <16 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = ashr <16 x i8> %a, %b ret <16 x i8> %shift @@ -364,6 +381,10 @@ define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v32i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %shift = ashr <32 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %shift = ashr <32 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift ; %shift = ashr <32 x i8> %a, %b ret <32 x i8> %shift @@ -397,6 +418,10 @@ define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = ashr <64 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = ashr <64 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = ashr <64 x i8> %a, %b ret <64 x i8> %shift @@ -834,6 +859,12 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, i16 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <32 x i16> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i16> undef, i16 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <32 x i16> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %insert = insertelement <32 x i16> undef, i16 %b, i32 0 %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer @@ -889,6 +920,12 @@ define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, i8 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <16 x i8> %insert, <16 x i8> undef, <16 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %shift = ashr <16 x i8> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <16 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <16 x i8> %insert, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %shift = ashr <16 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %insert = insertelement <16 x i8> undef, i8 %b, i32 0 %splat = shufflevector <16 x i8> %insert, <16 x i8> undef, <16 x i32> zeroinitializer @@ -944,6 +981,12 @@ define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, i8 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i8> %insert, <32 x i8> undef, <32 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %shift = ashr <32 x i8> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i8> %insert, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %shift = ashr <32 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift ; %insert = insertelement <32 x i8> undef, i8 %b, i32 0 %splat = shufflevector <32 x i8> %insert, <32 x i8> undef, <32 x i32> zeroinitializer @@ -999,6 +1042,12 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, i8 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %shift = ashr <64 x i8> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <64 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %shift = ashr <64 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %insert = insertelement <64 x i8> undef, i8 %b, i32 0 %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer @@ -1220,6 +1269,10 @@ define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v8i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift ; %shift = ashr <8 x i16> %a, ret <8 x i16> %shift @@ -1253,6 +1306,10 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = ashr <16 x i16> %a, ret <16 x i16> %shift @@ -1286,6 +1343,10 @@ define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = ashr <32 x i16> %a, ret <32 x i16> %shift @@ -1319,6 +1380,10 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = ashr <16 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = ashr <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = ashr <16 x i8> %a, ret <16 x i8> %shift @@ -1352,6 +1417,10 @@ define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v32i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %shift = ashr <32 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %shift = ashr <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift ; %shift = ashr <32 x i8> %a, ret <32 x i8> %shift @@ -1385,6 +1454,10 @@ define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = ashr <64 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = ashr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = ashr <64 x i8> %a, ret <64 x i8> %shift @@ -1649,6 +1722,10 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = ashr <16 x i16> %a, ret <16 x i16> %shift @@ -1686,6 +1763,10 @@ define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = ashr <32 x i16> %a, ret <32 x i16> %shift @@ -1723,6 +1804,10 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = ashr <16 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = ashr <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = ashr <16 x i8> %a, ret <16 x i8> %shift @@ -1760,6 +1845,10 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v32i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %shift = ashr <32 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %shift = ashr <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift ; %shift = ashr <32 x i8> %a, ret <32 x i8> %shift @@ -1797,6 +1886,10 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %shift = ashr <64 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %shift = ashr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = ashr <64 x i8> %a, ret <64 x i8> %shift diff --git a/llvm/test/Analysis/CostModel/X86/vshift-ashr-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/vshift-ashr-sizelatency.ll index 8286b3967af6..84ccad029415 100644 --- a/llvm/test/Analysis/CostModel/X86/vshift-ashr-sizelatency.ll +++ b/llvm/test/Analysis/CostModel/X86/vshift-ashr-sizelatency.ll @@ -15,6 +15,7 @@ ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mcpu=btver2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX1 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mcpu=tigerlake | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512GFNI ; Verify the cost of vector logical shift right instructions. @@ -232,6 +233,10 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v8i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift ; %shift = ashr <8 x i16> %a, %b ret <8 x i16> %shift @@ -265,6 +270,10 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = ashr <16 x i16> %a, %b ret <16 x i16> %shift @@ -298,6 +307,10 @@ define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = ashr <32 x i16> %a, %b ret <32 x i16> %shift @@ -331,6 +344,10 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = ashr <16 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = ashr <16 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = ashr <16 x i8> %a, %b ret <16 x i8> %shift @@ -364,6 +381,10 @@ define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v32i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %shift = ashr <32 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %shift = ashr <32 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift ; %shift = ashr <32 x i8> %a, %b ret <32 x i8> %shift @@ -397,6 +418,10 @@ define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %shift = ashr <64 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %shift = ashr <64 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = ashr <64 x i8> %a, %b ret <64 x i8> %shift @@ -786,6 +811,12 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, i16 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %shift = ashr <32 x i16> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i16> undef, i16 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %shift = ashr <32 x i16> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %insert = insertelement <32 x i16> undef, i16 %b, i32 0 %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer @@ -835,6 +866,12 @@ define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, i8 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <16 x i8> %insert, <16 x i8> undef, <16 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %shift = ashr <16 x i8> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <16 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <16 x i8> %insert, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %shift = ashr <16 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %insert = insertelement <16 x i8> undef, i8 %b, i32 0 %splat = shufflevector <16 x i8> %insert, <16 x i8> undef, <16 x i32> zeroinitializer @@ -890,6 +927,12 @@ define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, i8 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i8> %insert, <32 x i8> undef, <32 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %shift = ashr <32 x i8> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i8> %insert, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %shift = ashr <32 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift ; %insert = insertelement <32 x i8> undef, i8 %b, i32 0 %splat = shufflevector <32 x i8> %insert, <32 x i8> undef, <32 x i32> zeroinitializer @@ -945,6 +988,12 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, i8 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = ashr <64 x i8> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <64 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = ashr <64 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %insert = insertelement <64 x i8> undef, i8 %b, i32 0 %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer @@ -1166,6 +1215,10 @@ define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v8i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift ; %shift = ashr <8 x i16> %a, ret <8 x i16> %shift @@ -1199,6 +1252,10 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = ashr <16 x i16> %a, ret <16 x i16> %shift @@ -1232,6 +1289,10 @@ define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = ashr <32 x i16> %a, ret <32 x i16> %shift @@ -1265,6 +1326,10 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = ashr <16 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = ashr <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = ashr <16 x i8> %a, ret <16 x i8> %shift @@ -1298,6 +1363,10 @@ define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v32i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %shift = ashr <32 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %shift = ashr <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift ; %shift = ashr <32 x i8> %a, ret <32 x i8> %shift @@ -1331,6 +1400,10 @@ define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %shift = ashr <64 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %shift = ashr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = ashr <64 x i8> %a, ret <64 x i8> %shift @@ -1547,6 +1620,10 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = ashr <16 x i16> %a, ret <16 x i16> %shift @@ -1584,6 +1661,10 @@ define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = ashr <32 x i16> %a, ret <32 x i16> %shift @@ -1617,6 +1698,10 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = ashr <16 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = ashr <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = ashr <16 x i8> %a, ret <16 x i8> %shift @@ -1654,6 +1739,10 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v32i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = ashr <32 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = ashr <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift ; %shift = ashr <32 x i8> %a, ret <32 x i8> %shift @@ -1691,6 +1780,10 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shift = ashr <64 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shift = ashr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = ashr <64 x i8> %a, ret <64 x i8> %shift diff --git a/llvm/test/Analysis/CostModel/X86/vshift-lshr-codesize.ll b/llvm/test/Analysis/CostModel/X86/vshift-lshr-codesize.ll index 7c506af62e81..a0e15bb8ff73 100644 --- a/llvm/test/Analysis/CostModel/X86/vshift-lshr-codesize.ll +++ b/llvm/test/Analysis/CostModel/X86/vshift-lshr-codesize.ll @@ -15,6 +15,7 @@ ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mcpu=btver2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX1 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mcpu=tigerlake | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512GFNI ; Verify the cost of vector logical shift right instructions. @@ -240,6 +241,10 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v8i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift ; %shift = lshr <8 x i16> %a, %b ret <8 x i16> %shift @@ -273,6 +278,10 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = lshr <16 x i16> %a, %b ret <16 x i16> %shift @@ -306,6 +315,10 @@ define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = lshr <32 x i16> %a, %b ret <32 x i16> %shift @@ -335,6 +348,10 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <16 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <16 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = lshr <16 x i8> %a, %b ret <16 x i8> %shift @@ -397,6 +414,10 @@ define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = lshr <64 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = lshr <64 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = lshr <64 x i8> %a, %b ret <64 x i8> %shift @@ -762,6 +783,12 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, i16 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = lshr <32 x i16> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i16> undef, i16 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = lshr <32 x i16> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %insert = insertelement <32 x i16> undef, i16 %b, i32 0 %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer @@ -909,6 +936,12 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, i8 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %shift = lshr <64 x i8> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <64 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %shift = lshr <64 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %insert = insertelement <64 x i8> undef, i8 %b, i32 0 %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer @@ -1138,6 +1171,10 @@ define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v8i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift ; %shift = lshr <8 x i16> %a, ret <8 x i16> %shift @@ -1171,6 +1208,10 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = lshr <16 x i16> %a, ret <16 x i16> %shift @@ -1204,6 +1245,10 @@ define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = lshr <32 x i16> %a, ret <32 x i16> %shift @@ -1233,6 +1278,10 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <16 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = lshr <16 x i8> %a, ret <16 x i8> %shift @@ -1295,6 +1344,10 @@ define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = lshr <64 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = lshr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = lshr <64 x i8> %a, ret <64 x i8> %shift @@ -1495,6 +1548,10 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = lshr <16 x i16> %a, ret <16 x i16> %shift @@ -1532,6 +1589,10 @@ define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = lshr <32 x i16> %a, ret <32 x i16> %shift @@ -1631,6 +1692,10 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = lshr <64 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = lshr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = lshr <64 x i8> %a, ret <64 x i8> %shift diff --git a/llvm/test/Analysis/CostModel/X86/vshift-lshr-cost-inseltpoison.ll b/llvm/test/Analysis/CostModel/X86/vshift-lshr-cost-inseltpoison.ll index 855d2b8e88e3..34ea1a644f6c 100644 --- a/llvm/test/Analysis/CostModel/X86/vshift-lshr-cost-inseltpoison.ll +++ b/llvm/test/Analysis/CostModel/X86/vshift-lshr-cost-inseltpoison.ll @@ -15,6 +15,7 @@ ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=btver2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX1 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=tigerlake | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512GFNI ; Verify the cost of vector logical shift right instructions. @@ -232,6 +233,10 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; AVX512BWVL-LABEL: 'var_shift_v8i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift ; %shift = lshr <8 x i16> %a, %b ret <8 x i16> %shift @@ -273,6 +278,10 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; AVX512BWVL-LABEL: 'var_shift_v16i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = lshr <16 x i16> %a, %b ret <16 x i16> %shift @@ -314,6 +323,10 @@ define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) { ; AVX512BWVL-LABEL: 'var_shift_v32i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = lshr <32 x i16> %a, %b ret <32 x i16> %shift @@ -355,6 +368,10 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; AVX512BWVL-LABEL: 'var_shift_v16i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <16 x i8> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <16 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift ; %shift = lshr <16 x i8> %a, %b ret <16 x i8> %shift @@ -396,6 +413,10 @@ define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { ; AVX512BWVL-LABEL: 'var_shift_v32i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <32 x i8> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <32 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = lshr <32 x i8> %a, %b ret <32 x i8> %shift @@ -437,6 +458,10 @@ define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) { ; AVX512BWVL-LABEL: 'var_shift_v64i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %shift = lshr <64 x i8> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %shift = lshr <64 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = lshr <64 x i8> %a, %b ret <64 x i8> %shift @@ -826,6 +851,12 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, i16 %b) { ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> poison, <32 x i32> zeroinitializer ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = lshr <32 x i16> %a, %splat ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i16> poison, i16 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> poison, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = lshr <32 x i16> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %insert = insertelement <32 x i16> poison, i16 %b, i32 0 %splat = shufflevector <32 x i16> %insert, <32 x i16> poison, <32 x i32> zeroinitializer @@ -991,6 +1022,12 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, i8 %b) { ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> poison, <64 x i32> zeroinitializer ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <64 x i8> %a, %splat ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <64 x i8> poison, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> poison, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <64 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %insert = insertelement <64 x i8> poison, i8 %b, i32 0 %splat = shufflevector <64 x i8> %insert, <64 x i8> poison, <64 x i32> zeroinitializer @@ -1200,6 +1237,10 @@ define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v8i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift ; %shift = lshr <8 x i16> %a, ret <8 x i16> %shift @@ -1241,6 +1282,10 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v16i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = lshr <16 x i16> %a, ret <16 x i16> %shift @@ -1282,6 +1327,10 @@ define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v32i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = lshr <32 x i16> %a, ret <32 x i16> %shift @@ -1323,6 +1372,10 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v16i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <16 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift ; %shift = lshr <16 x i8> %a, ret <16 x i8> %shift @@ -1364,6 +1417,10 @@ define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v32i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <32 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = lshr <32 x i8> %a, ret <32 x i8> %shift @@ -1405,6 +1462,10 @@ define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v64i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %shift = lshr <64 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %shift = lshr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = lshr <64 x i8> %a, ret <64 x i8> %shift @@ -1593,6 +1654,10 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v16i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = lshr <16 x i16> %a, ret <16 x i16> %shift @@ -1634,6 +1699,10 @@ define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v32i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = lshr <32 x i16> %a, ret <32 x i16> %shift @@ -1700,6 +1769,10 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v32i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = lshr <32 x i8> %a, ret <32 x i8> %shift @@ -1741,6 +1814,10 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v64i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <64 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = lshr <64 x i8> %a, ret <64 x i8> %shift diff --git a/llvm/test/Analysis/CostModel/X86/vshift-lshr-cost.ll b/llvm/test/Analysis/CostModel/X86/vshift-lshr-cost.ll index 55894dc93295..93b844eddf18 100644 --- a/llvm/test/Analysis/CostModel/X86/vshift-lshr-cost.ll +++ b/llvm/test/Analysis/CostModel/X86/vshift-lshr-cost.ll @@ -15,6 +15,7 @@ ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=btver2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX1 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=tigerlake | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512GFNI ; Verify the cost of vector logical shift right instructions. @@ -232,6 +233,10 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; AVX512BWVL-LABEL: 'var_shift_v8i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift ; %shift = lshr <8 x i16> %a, %b ret <8 x i16> %shift @@ -273,6 +278,10 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; AVX512BWVL-LABEL: 'var_shift_v16i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = lshr <16 x i16> %a, %b ret <16 x i16> %shift @@ -314,6 +323,10 @@ define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) { ; AVX512BWVL-LABEL: 'var_shift_v32i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = lshr <32 x i16> %a, %b ret <32 x i16> %shift @@ -355,6 +368,10 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; AVX512BWVL-LABEL: 'var_shift_v16i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <16 x i8> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <16 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift ; %shift = lshr <16 x i8> %a, %b ret <16 x i8> %shift @@ -396,6 +413,10 @@ define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { ; AVX512BWVL-LABEL: 'var_shift_v32i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <32 x i8> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <32 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = lshr <32 x i8> %a, %b ret <32 x i8> %shift @@ -437,6 +458,10 @@ define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) { ; AVX512BWVL-LABEL: 'var_shift_v64i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %shift = lshr <64 x i8> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %shift = lshr <64 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = lshr <64 x i8> %a, %b ret <64 x i8> %shift @@ -826,6 +851,12 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, i16 %b) { ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = lshr <32 x i16> %a, %splat ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i16> undef, i16 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = lshr <32 x i16> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %insert = insertelement <32 x i16> undef, i16 %b, i32 0 %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer @@ -991,6 +1022,12 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, i8 %b) { ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <64 x i8> %a, %splat ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <64 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <64 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %insert = insertelement <64 x i8> undef, i8 %b, i32 0 %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer @@ -1200,6 +1237,10 @@ define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v8i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift ; %shift = lshr <8 x i16> %a, ret <8 x i16> %shift @@ -1241,6 +1282,10 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v16i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = lshr <16 x i16> %a, ret <16 x i16> %shift @@ -1282,6 +1327,10 @@ define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v32i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = lshr <32 x i16> %a, ret <32 x i16> %shift @@ -1323,6 +1372,10 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v16i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <16 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift ; %shift = lshr <16 x i8> %a, ret <16 x i8> %shift @@ -1364,6 +1417,10 @@ define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v32i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <32 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = lshr <32 x i8> %a, ret <32 x i8> %shift @@ -1405,6 +1462,10 @@ define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v64i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %shift = lshr <64 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %shift = lshr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = lshr <64 x i8> %a, ret <64 x i8> %shift @@ -1593,6 +1654,10 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v16i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = lshr <16 x i16> %a, ret <16 x i16> %shift @@ -1634,6 +1699,10 @@ define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v32i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = lshr <32 x i16> %a, ret <32 x i16> %shift @@ -1700,6 +1769,10 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v32i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = lshr <32 x i8> %a, ret <32 x i8> %shift @@ -1741,6 +1814,10 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v64i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <64 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = lshr <64 x i8> %a, ret <64 x i8> %shift diff --git a/llvm/test/Analysis/CostModel/X86/vshift-lshr-latency.ll b/llvm/test/Analysis/CostModel/X86/vshift-lshr-latency.ll index 92fe253d38e7..61620e2cc97e 100644 --- a/llvm/test/Analysis/CostModel/X86/vshift-lshr-latency.ll +++ b/llvm/test/Analysis/CostModel/X86/vshift-lshr-latency.ll @@ -15,6 +15,7 @@ ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mcpu=slm | FileCheck %s --check-prefixes=SSE,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mcpu=goldmont | FileCheck %s --check-prefixes=SSE,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mcpu=btver2 | FileCheck %s --check-prefixes=AVX1 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mcpu=tigerlake | FileCheck %s --check-prefixes=AVX512,AVX512GFNI ; Verify the cost of vector logical shift right instructions. @@ -240,6 +241,10 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v8i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift ; %shift = lshr <8 x i16> %a, %b ret <8 x i16> %shift @@ -277,6 +282,10 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = lshr <16 x i16> %a, %b ret <16 x i16> %shift @@ -314,6 +323,10 @@ define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = lshr <32 x i16> %a, %b ret <32 x i16> %shift @@ -351,6 +364,10 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = lshr <16 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = lshr <16 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = lshr <16 x i8> %a, %b ret <16 x i8> %shift @@ -421,6 +438,10 @@ define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 27 for instruction: %shift = lshr <64 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 27 for instruction: %shift = lshr <64 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = lshr <64 x i8> %a, %b ret <64 x i8> %shift @@ -834,6 +855,12 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, i16 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <32 x i16> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i16> undef, i16 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <32 x i16> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %insert = insertelement <32 x i16> undef, i16 %b, i32 0 %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer @@ -889,6 +916,12 @@ define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, i8 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <16 x i8> %insert, <16 x i8> undef, <16 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %shift = lshr <16 x i8> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <16 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <16 x i8> %insert, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %shift = lshr <16 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %insert = insertelement <16 x i8> undef, i8 %b, i32 0 %splat = shufflevector <16 x i8> %insert, <16 x i8> undef, <16 x i32> zeroinitializer @@ -993,6 +1026,12 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, i8 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = lshr <64 x i8> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <64 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = lshr <64 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %insert = insertelement <64 x i8> undef, i8 %b, i32 0 %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer @@ -1222,6 +1261,10 @@ define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v8i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift ; %shift = lshr <8 x i16> %a, ret <8 x i16> %shift @@ -1259,6 +1302,10 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = lshr <16 x i16> %a, ret <16 x i16> %shift @@ -1296,6 +1343,10 @@ define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = lshr <32 x i16> %a, ret <32 x i16> %shift @@ -1333,6 +1384,10 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = lshr <16 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = lshr <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = lshr <16 x i8> %a, ret <16 x i8> %shift @@ -1403,6 +1458,10 @@ define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 27 for instruction: %shift = lshr <64 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 27 for instruction: %shift = lshr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = lshr <64 x i8> %a, ret <64 x i8> %shift @@ -1643,6 +1702,10 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = lshr <16 x i16> %a, ret <16 x i16> %shift @@ -1676,6 +1739,10 @@ define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = lshr <32 x i16> %a, ret <32 x i16> %shift @@ -1709,6 +1776,10 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %shift = lshr <16 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %shift = lshr <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = lshr <16 x i8> %a, ret <16 x i8> %shift @@ -1771,6 +1842,10 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = lshr <64 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = lshr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = lshr <64 x i8> %a, ret <64 x i8> %shift diff --git a/llvm/test/Analysis/CostModel/X86/vshift-lshr-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/vshift-lshr-sizelatency.ll index fe8a7d91cc37..e6b6ac75b65d 100644 --- a/llvm/test/Analysis/CostModel/X86/vshift-lshr-sizelatency.ll +++ b/llvm/test/Analysis/CostModel/X86/vshift-lshr-sizelatency.ll @@ -15,6 +15,7 @@ ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mcpu=btver2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX1 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mcpu=tigerlake | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512GFNI ; Verify the cost of vector logical shift right instructions. @@ -240,6 +241,10 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v8i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift ; %shift = lshr <8 x i16> %a, %b ret <8 x i16> %shift @@ -273,6 +278,10 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = lshr <16 x i16> %a, %b ret <16 x i16> %shift @@ -306,6 +315,10 @@ define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = lshr <32 x i16> %a, %b ret <32 x i16> %shift @@ -335,6 +348,10 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = lshr <16 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = lshr <16 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = lshr <16 x i8> %a, %b ret <16 x i8> %shift @@ -368,6 +385,10 @@ define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v32i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %shift = lshr <32 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %shift = lshr <32 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift ; %shift = lshr <32 x i8> %a, %b ret <32 x i8> %shift @@ -401,6 +422,10 @@ define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %shift = lshr <64 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %shift = lshr <64 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = lshr <64 x i8> %a, %b ret <64 x i8> %shift @@ -766,6 +791,12 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, i16 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %shift = lshr <32 x i16> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i16> undef, i16 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %shift = lshr <32 x i16> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %insert = insertelement <32 x i16> undef, i16 %b, i32 0 %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer @@ -913,6 +944,12 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, i8 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %shift = lshr <64 x i8> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <64 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %shift = lshr <64 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %insert = insertelement <64 x i8> undef, i8 %b, i32 0 %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer @@ -1142,6 +1179,10 @@ define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v8i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift ; %shift = lshr <8 x i16> %a, ret <8 x i16> %shift @@ -1175,6 +1216,10 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = lshr <16 x i16> %a, ret <16 x i16> %shift @@ -1208,6 +1253,10 @@ define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = lshr <32 x i16> %a, ret <32 x i16> %shift @@ -1237,6 +1286,10 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = lshr <16 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = lshr <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = lshr <16 x i8> %a, ret <16 x i8> %shift @@ -1270,6 +1323,10 @@ define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v32i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %shift = lshr <32 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %shift = lshr <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift ; %shift = lshr <32 x i8> %a, ret <32 x i8> %shift @@ -1303,6 +1360,10 @@ define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %shift = lshr <64 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %shift = lshr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = lshr <64 x i8> %a, ret <64 x i8> %shift @@ -1503,6 +1564,10 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = lshr <16 x i16> %a, ret <16 x i16> %shift @@ -1540,6 +1605,10 @@ define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = lshr <32 x i16> %a, ret <32 x i16> %shift @@ -1606,6 +1675,10 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v32i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %shift = lshr <32 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %shift = lshr <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift ; %shift = lshr <32 x i8> %a, ret <32 x i8> %shift @@ -1643,6 +1716,10 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %shift = lshr <64 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %shift = lshr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = lshr <64 x i8> %a, ret <64 x i8> %shift diff --git a/llvm/test/Analysis/CostModel/X86/vshift-shl-codesize.ll b/llvm/test/Analysis/CostModel/X86/vshift-shl-codesize.ll index 07229e22873c..265658b1e3a2 100644 --- a/llvm/test/Analysis/CostModel/X86/vshift-shl-codesize.ll +++ b/llvm/test/Analysis/CostModel/X86/vshift-shl-codesize.ll @@ -15,6 +15,7 @@ ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mcpu=btver2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX1 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mcpu=tigerlake | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512GFNI ; Verify the cost of vector logical shift right instructions. @@ -240,6 +241,10 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v8i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <8 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <8 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift ; %shift = shl <8 x i16> %a, %b ret <8 x i16> %shift @@ -273,6 +278,10 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = shl <16 x i16> %a, %b ret <16 x i16> %shift @@ -306,6 +315,10 @@ define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = shl <32 x i16> %a, %b ret <32 x i16> %shift @@ -335,6 +348,10 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <16 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <16 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = shl <16 x i8> %a, %b ret <16 x i8> %shift @@ -397,6 +414,10 @@ define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %shift = shl <64 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %shift = shl <64 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = shl <64 x i8> %a, %b ret <64 x i8> %shift @@ -762,6 +783,12 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, i16 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = shl <32 x i16> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i16> undef, i16 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = shl <32 x i16> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %insert = insertelement <32 x i16> undef, i16 %b, i32 0 %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer @@ -915,6 +942,12 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, i8 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shift = shl <64 x i8> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <64 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shift = shl <64 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %insert = insertelement <64 x i8> undef, i8 %b, i32 0 %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer @@ -1186,6 +1219,10 @@ define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = shl <32 x i16> %a, ret <32 x i16> %shift @@ -1215,6 +1252,10 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <16 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = shl <16 x i8> %a, ret <16 x i8> %shift @@ -1277,6 +1318,10 @@ define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %shift = shl <64 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %shift = shl <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = shl <64 x i8> %a, ret <64 x i8> %shift @@ -1477,6 +1522,10 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = shl <16 x i16> %a, ret <16 x i16> %shift @@ -1514,6 +1563,10 @@ define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = shl <32 x i16> %a, ret <32 x i16> %shift @@ -1613,6 +1666,10 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = shl <64 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = shl <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = shl <64 x i8> %a, ret <64 x i8> %shift diff --git a/llvm/test/Analysis/CostModel/X86/vshift-shl-cost-inseltpoison.ll b/llvm/test/Analysis/CostModel/X86/vshift-shl-cost-inseltpoison.ll index d8737d236564..f093d1c8ca35 100644 --- a/llvm/test/Analysis/CostModel/X86/vshift-shl-cost-inseltpoison.ll +++ b/llvm/test/Analysis/CostModel/X86/vshift-shl-cost-inseltpoison.ll @@ -15,6 +15,7 @@ ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE,SLM ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=btver2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX1 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=tigerlake | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512GFNI ; Verify the cost of vector shift left instructions. @@ -236,6 +237,10 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; SLM-LABEL: 'var_shift_v8i16' ; SLM-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %shift = shl <8 x i16> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <8 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift ; %shift = shl <8 x i16> %a, %b ret <8 x i16> %shift @@ -281,6 +286,10 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; SLM-LABEL: 'var_shift_v16i16' ; SLM-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %shift = shl <16 x i16> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = shl <16 x i16> %a, %b ret <16 x i16> %shift @@ -326,6 +335,10 @@ define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) { ; SLM-LABEL: 'var_shift_v32i16' ; SLM-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %shift = shl <32 x i16> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = shl <32 x i16> %a, %b ret <32 x i16> %shift @@ -371,6 +384,10 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; SLM-LABEL: 'var_shift_v16i8' ; SLM-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = shl <16 x i8> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <16 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift ; %shift = shl <16 x i8> %a, %b ret <16 x i8> %shift @@ -416,6 +433,10 @@ define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { ; SLM-LABEL: 'var_shift_v32i8' ; SLM-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %shift = shl <32 x i8> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <32 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = shl <32 x i8> %a, %b ret <32 x i8> %shift @@ -461,6 +482,10 @@ define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) { ; SLM-LABEL: 'var_shift_v64i8' ; SLM-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %shift = shl <64 x i8> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shift = shl <64 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = shl <64 x i8> %a, %b ret <64 x i8> %shift @@ -868,6 +893,12 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, i16 %b) { ; SLM-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> poison, <32 x i32> zeroinitializer ; SLM-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = shl <32 x i16> %a, %splat ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i16> poison, i16 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> poison, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = shl <32 x i16> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %insert = insertelement <32 x i16> poison, i16 %b, i32 0 %splat = shufflevector <32 x i16> %insert, <32 x i16> poison, <32 x i32> zeroinitializer @@ -1051,6 +1082,12 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, i8 %b) { ; SLM-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> poison, <64 x i32> zeroinitializer ; SLM-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %shift = shl <64 x i8> %a, %splat ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <64 x i8> poison, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> poison, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <64 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %insert = insertelement <64 x i8> poison, i8 %b, i32 0 %splat = shufflevector <64 x i8> %insert, <64 x i8> poison, <64 x i32> zeroinitializer @@ -1305,6 +1342,10 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) { ; SLM-LABEL: 'constant_shift_v16i16' ; SLM-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <16 x i16> %a, ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = shl <16 x i16> %a, ret <16 x i16> %shift @@ -1354,6 +1395,10 @@ define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) { ; SLM-LABEL: 'constant_shift_v32i16' ; SLM-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = shl <32 x i16> %a, ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = shl <32 x i16> %a, ret <32 x i16> %shift @@ -1399,6 +1444,10 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; SLM-LABEL: 'constant_shift_v16i8' ; SLM-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = shl <16 x i8> %a, ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift ; %shift = shl <16 x i8> %a, ret <16 x i8> %shift @@ -1444,6 +1493,10 @@ define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) { ; SLM-LABEL: 'constant_shift_v32i8' ; SLM-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %shift = shl <32 x i8> %a, ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = shl <32 x i8> %a, ret <32 x i8> %shift @@ -1489,6 +1542,10 @@ define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) { ; SLM-LABEL: 'constant_shift_v64i8' ; SLM-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %shift = shl <64 x i8> %a, ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shift = shl <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = shl <64 x i8> %a, ret <64 x i8> %shift @@ -1677,6 +1734,10 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v16i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = shl <16 x i16> %a, ret <16 x i16> %shift @@ -1718,6 +1779,10 @@ define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v32i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = shl <32 x i16> %a, ret <32 x i16> %shift @@ -1784,6 +1849,10 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v32i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = shl <32 x i8> %a, ret <32 x i8> %shift @@ -1825,6 +1894,10 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v64i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <64 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = shl <64 x i8> %a, ret <64 x i8> %shift @@ -2044,6 +2117,10 @@ define <16 x i16> @test6(<16 x i16> %a) { ; SLM-LABEL: 'test6' ; SLM-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shl = shl <16 x i16> %a, ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shl +; +; AVX512GFNI-LABEL: 'test6' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shl = shl <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shl ; %shl = shl <16 x i16> %a, ret <16 x i16> %shl @@ -2162,6 +2239,10 @@ define <32 x i16> @test9(<32 x i16> %a) { ; SLM-LABEL: 'test9' ; SLM-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shl = shl <32 x i16> %a, ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shl +; +; AVX512GFNI-LABEL: 'test9' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shl = shl <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shl ; %shl = shl <32 x i16> %a, ret <32 x i16> %shl diff --git a/llvm/test/Analysis/CostModel/X86/vshift-shl-cost.ll b/llvm/test/Analysis/CostModel/X86/vshift-shl-cost.ll index b254ff27dacc..09521cfa2d23 100644 --- a/llvm/test/Analysis/CostModel/X86/vshift-shl-cost.ll +++ b/llvm/test/Analysis/CostModel/X86/vshift-shl-cost.ll @@ -15,6 +15,7 @@ ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE,SLM ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=btver2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX1 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=tigerlake | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512GFNI ; Verify the cost of vector shift left instructions. @@ -236,6 +237,10 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; SLM-LABEL: 'var_shift_v8i16' ; SLM-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %shift = shl <8 x i16> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <8 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift ; %shift = shl <8 x i16> %a, %b ret <8 x i16> %shift @@ -281,6 +286,10 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; SLM-LABEL: 'var_shift_v16i16' ; SLM-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %shift = shl <16 x i16> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = shl <16 x i16> %a, %b ret <16 x i16> %shift @@ -326,6 +335,10 @@ define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) { ; SLM-LABEL: 'var_shift_v32i16' ; SLM-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %shift = shl <32 x i16> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = shl <32 x i16> %a, %b ret <32 x i16> %shift @@ -371,6 +384,10 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; SLM-LABEL: 'var_shift_v16i8' ; SLM-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = shl <16 x i8> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <16 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift ; %shift = shl <16 x i8> %a, %b ret <16 x i8> %shift @@ -416,6 +433,10 @@ define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { ; SLM-LABEL: 'var_shift_v32i8' ; SLM-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %shift = shl <32 x i8> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <32 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = shl <32 x i8> %a, %b ret <32 x i8> %shift @@ -461,6 +482,10 @@ define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) { ; SLM-LABEL: 'var_shift_v64i8' ; SLM-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %shift = shl <64 x i8> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shift = shl <64 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = shl <64 x i8> %a, %b ret <64 x i8> %shift @@ -868,6 +893,12 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, i16 %b) { ; SLM-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer ; SLM-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = shl <32 x i16> %a, %splat ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i16> undef, i16 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = shl <32 x i16> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %insert = insertelement <32 x i16> undef, i16 %b, i32 0 %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer @@ -1051,6 +1082,12 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, i8 %b) { ; SLM-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer ; SLM-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %shift = shl <64 x i8> %a, %splat ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <64 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <64 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %insert = insertelement <64 x i8> undef, i8 %b, i32 0 %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer @@ -1305,6 +1342,10 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) { ; SLM-LABEL: 'constant_shift_v16i16' ; SLM-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <16 x i16> %a, ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = shl <16 x i16> %a, ret <16 x i16> %shift @@ -1354,6 +1395,10 @@ define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) { ; SLM-LABEL: 'constant_shift_v32i16' ; SLM-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = shl <32 x i16> %a, ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = shl <32 x i16> %a, ret <32 x i16> %shift @@ -1399,6 +1444,10 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; SLM-LABEL: 'constant_shift_v16i8' ; SLM-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = shl <16 x i8> %a, ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift ; %shift = shl <16 x i8> %a, ret <16 x i8> %shift @@ -1444,6 +1493,10 @@ define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) { ; SLM-LABEL: 'constant_shift_v32i8' ; SLM-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %shift = shl <32 x i8> %a, ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = shl <32 x i8> %a, ret <32 x i8> %shift @@ -1489,6 +1542,10 @@ define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) { ; SLM-LABEL: 'constant_shift_v64i8' ; SLM-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %shift = shl <64 x i8> %a, ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shift = shl <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = shl <64 x i8> %a, ret <64 x i8> %shift @@ -1677,6 +1734,10 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v16i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = shl <16 x i16> %a, ret <16 x i16> %shift @@ -1718,6 +1779,10 @@ define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v32i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = shl <32 x i16> %a, ret <32 x i16> %shift @@ -1784,6 +1849,10 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v32i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = shl <32 x i8> %a, ret <32 x i8> %shift @@ -1825,6 +1894,10 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v64i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <64 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = shl <64 x i8> %a, ret <64 x i8> %shift @@ -2044,6 +2117,10 @@ define <16 x i16> @test6(<16 x i16> %a) { ; SLM-LABEL: 'test6' ; SLM-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shl = shl <16 x i16> %a, ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shl +; +; AVX512GFNI-LABEL: 'test6' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shl = shl <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shl ; %shl = shl <16 x i16> %a, ret <16 x i16> %shl @@ -2162,6 +2239,10 @@ define <32 x i16> @test9(<32 x i16> %a) { ; SLM-LABEL: 'test9' ; SLM-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shl = shl <32 x i16> %a, ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shl +; +; AVX512GFNI-LABEL: 'test9' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shl = shl <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shl ; %shl = shl <32 x i16> %a, ret <32 x i16> %shl diff --git a/llvm/test/Analysis/CostModel/X86/vshift-shl-latency.ll b/llvm/test/Analysis/CostModel/X86/vshift-shl-latency.ll index c53dbc5f586a..42c91144ff6f 100644 --- a/llvm/test/Analysis/CostModel/X86/vshift-shl-latency.ll +++ b/llvm/test/Analysis/CostModel/X86/vshift-shl-latency.ll @@ -15,6 +15,7 @@ ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mcpu=slm | FileCheck %s --check-prefixes=SSE,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mcpu=goldmont | FileCheck %s --check-prefixes=SSE,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mcpu=btver2 | FileCheck %s --check-prefixes=AVX1 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mcpu=tigerlake | FileCheck %s --check-prefixes=AVX512,AVX512GFNI ; Verify the cost of vector logical shift right instructions. @@ -240,6 +241,10 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v8i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <8 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <8 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift ; %shift = shl <8 x i16> %a, %b ret <8 x i16> %shift @@ -277,6 +282,10 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = shl <16 x i16> %a, %b ret <16 x i16> %shift @@ -314,6 +323,10 @@ define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = shl <32 x i16> %a, %b ret <32 x i16> %shift @@ -351,6 +364,10 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = shl <16 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = shl <16 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = shl <16 x i8> %a, %b ret <16 x i8> %shift @@ -830,6 +847,12 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, i16 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <32 x i16> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i16> undef, i16 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <32 x i16> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %insert = insertelement <32 x i16> undef, i16 %b, i32 0 %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer @@ -983,6 +1006,12 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, i8 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %shift = shl <64 x i8> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <64 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %shift = shl <64 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %insert = insertelement <64 x i8> undef, i8 %b, i32 0 %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer @@ -1208,6 +1237,10 @@ define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v8i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <8 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <8 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift ; %shift = shl <8 x i16> %a, ret <8 x i16> %shift @@ -1241,6 +1274,10 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = shl <16 x i16> %a, ret <16 x i16> %shift @@ -1274,6 +1311,10 @@ define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = shl <32 x i16> %a, ret <32 x i16> %shift @@ -1311,6 +1352,10 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = shl <16 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = shl <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = shl <16 x i8> %a, ret <16 x i8> %shift @@ -1617,6 +1662,10 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = shl <16 x i16> %a, ret <16 x i16> %shift @@ -1650,6 +1699,10 @@ define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = shl <32 x i16> %a, ret <32 x i16> %shift @@ -1683,6 +1736,10 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %shift = shl <16 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %shift = shl <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = shl <16 x i8> %a, ret <16 x i8> %shift @@ -1745,6 +1802,10 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = shl <64 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = shl <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = shl <64 x i8> %a, ret <64 x i8> %shift diff --git a/llvm/test/Analysis/CostModel/X86/vshift-shl-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/vshift-shl-sizelatency.ll index cbf0f24b90fb..47b24df063ef 100644 --- a/llvm/test/Analysis/CostModel/X86/vshift-shl-sizelatency.ll +++ b/llvm/test/Analysis/CostModel/X86/vshift-shl-sizelatency.ll @@ -15,6 +15,7 @@ ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE,SLM ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mcpu=btver2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX1 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mcpu=tigerlake | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512GFNI ; Verify the cost of vector logical shift right instructions. @@ -263,6 +264,10 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; SLM-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %shift = shl <8 x i16> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift ; +; AVX512GFNI-LABEL: 'var_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <8 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift +; ; BTVER2-LABEL: 'var_shift_v8i16' ; BTVER2-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %shift = shl <8 x i16> %a, %b ; BTVER2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift @@ -303,6 +308,10 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; SLM-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %shift = shl <16 x i16> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; +; AVX512GFNI-LABEL: 'var_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; ; BTVER2-LABEL: 'var_shift_v16i16' ; BTVER2-NEXT: Cost Model: Found an estimated cost of 25 for instruction: %shift = shl <16 x i16> %a, %b ; BTVER2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift @@ -343,6 +352,10 @@ define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) { ; SLM-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %shift = shl <32 x i16> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; +; AVX512GFNI-LABEL: 'var_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; ; BTVER2-LABEL: 'var_shift_v32i16' ; BTVER2-NEXT: Cost Model: Found an estimated cost of 50 for instruction: %shift = shl <32 x i16> %a, %b ; BTVER2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift @@ -383,6 +396,10 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; SLM-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %shift = shl <16 x i8> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; +; AVX512GFNI-LABEL: 'var_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = shl <16 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; ; BTVER2-LABEL: 'var_shift_v16i8' ; BTVER2-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %shift = shl <16 x i8> %a, %b ; BTVER2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift @@ -423,6 +440,10 @@ define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { ; SLM-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %shift = shl <32 x i8> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift ; +; AVX512GFNI-LABEL: 'var_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %shift = shl <32 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift +; ; BTVER2-LABEL: 'var_shift_v32i8' ; BTVER2-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %shift = shl <32 x i8> %a, %b ; BTVER2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift @@ -463,6 +484,10 @@ define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) { ; SLM-NEXT: Cost Model: Found an estimated cost of 88 for instruction: %shift = shl <64 x i8> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; +; AVX512GFNI-LABEL: 'var_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = shl <64 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; ; BTVER2-LABEL: 'var_shift_v64i8' ; BTVER2-NEXT: Cost Model: Found an estimated cost of 80 for instruction: %shift = shl <64 x i8> %a, %b ; BTVER2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift @@ -824,6 +849,12 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, i16 %b) { ; SLM-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer ; SLM-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = shl <32 x i16> %a, %splat ; SLM-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i16> undef, i16 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %shift = shl <32 x i16> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %insert = insertelement <32 x i16> undef, i16 %b, i32 0 %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer @@ -995,6 +1026,12 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, i8 %b) { ; SLM-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer ; SLM-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %shift = shl <64 x i8> %a, %splat ; SLM-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <64 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = shl <64 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %insert = insertelement <64 x i8> undef, i8 %b, i32 0 %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer @@ -1238,6 +1275,10 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = shl <16 x i16> %a, ret <16 x i16> %shift @@ -1271,6 +1312,10 @@ define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = shl <32 x i16> %a, ret <32 x i16> %shift @@ -1309,6 +1354,10 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; SLM-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %shift = shl <16 x i8> %a, ; SLM-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; +; AVX512GFNI-LABEL: 'constant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = shl <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; ; BTVER2-LABEL: 'constant_shift_v16i8' ; BTVER2-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %shift = shl <16 x i8> %a, ; BTVER2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift @@ -1349,6 +1398,10 @@ define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) { ; SLM-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %shift = shl <32 x i8> %a, ; SLM-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift ; +; AVX512GFNI-LABEL: 'constant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %shift = shl <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift +; ; BTVER2-LABEL: 'constant_shift_v32i8' ; BTVER2-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %shift = shl <32 x i8> %a, ; BTVER2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift @@ -1389,6 +1442,10 @@ define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) { ; SLM-NEXT: Cost Model: Found an estimated cost of 88 for instruction: %shift = shl <64 x i8> %a, ; SLM-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; +; AVX512GFNI-LABEL: 'constant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = shl <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; ; BTVER2-LABEL: 'constant_shift_v64i8' ; BTVER2-NEXT: Cost Model: Found an estimated cost of 80 for instruction: %shift = shl <64 x i8> %a, ; BTVER2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift @@ -1571,6 +1628,10 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = shl <16 x i16> %a, ret <16 x i16> %shift @@ -1604,6 +1665,10 @@ define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = shl <32 x i16> %a, ret <32 x i16> %shift @@ -1662,6 +1727,10 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v32i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %shift = shl <32 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %shift = shl <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift ; %shift = shl <32 x i8> %a, ret <32 x i8> %shift @@ -1695,6 +1764,10 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %shift = shl <64 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %shift = shl <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = shl <64 x i8> %a, ret <64 x i8> %shift -- GitLab From da04e4afd3cae13581cac85688fbf10a5848655f Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Wed, 17 Apr 2024 12:05:26 -0500 Subject: [PATCH 081/862] [InstCombine] Use `auto *` instead of `auto` in `visitSIToFP`; NFC --- llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp index d242d3f443de..4537a47da2ce 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp @@ -1990,7 +1990,7 @@ Instruction *InstCombinerImpl::visitSIToFP(CastInst &CI) { if (Instruction *R = commonCastTransforms(CI)) return R; if (isKnownNonNegative(CI.getOperand(0), SQ)) { - auto UI = + auto *UI = CastInst::Create(Instruction::UIToFP, CI.getOperand(0), CI.getType()); UI->setNonNeg(true); return UI; -- GitLab From d423d80e560d8bf7ca493596d9f34a9e1f0eede7 Mon Sep 17 00:00:00 2001 From: Louis Dionne Date: Wed, 17 Apr 2024 13:36:53 -0400 Subject: [PATCH 082/862] [libc++][pstl] Promote CPU backends to top-level backends (#88968) This patch removes the two-level backend dispatching mechanism we had in the PSTL. Instead of selecting both a PSTL backend and a PSTL CPU backend, we now only select a top-level PSTL backend. This greatly simplifies the PSTL configuration layer. While this patch technically removes some flexibility from the PSTL configuration mechanism because CPU backends are not considered separately, it opens the door to a much more powerful configuration mechanism based on chained backends in a follow-up patch. This is a step towards overhauling the PSTL dispatching mechanism. --- libcxx/CMakeLists.txt | 18 +++---- libcxx/cmake/caches/Apple.cmake | 2 +- libcxx/include/CMakeLists.txt | 11 ++--- .../__algorithm/pstl_backends/cpu_backend.h | 23 --------- .../pstl_backends/cpu_backends/any_of.h | 2 +- .../pstl_backends/cpu_backends/backend.h | 45 ------------------ .../pstl_backends/cpu_backends/fill.h | 2 +- .../pstl_backends/cpu_backends/find_if.h | 2 +- .../pstl_backends/cpu_backends/for_each.h | 2 +- .../pstl_backends/cpu_backends/merge.h | 2 +- .../pstl_backends/cpu_backends/stable_sort.h | 2 +- .../pstl_backends/cpu_backends/transform.h | 2 +- .../cpu_backends/transform_reduce.h | 2 +- libcxx/include/__algorithm/pstl_copy.h | 2 +- libcxx/include/__algorithm/pstl_count.h | 2 +- libcxx/include/__algorithm/pstl_find.h | 2 +- libcxx/include/__algorithm/pstl_for_each.h | 2 +- libcxx/include/__algorithm/pstl_generate.h | 2 +- .../include/__algorithm/pstl_is_partitioned.h | 2 +- libcxx/include/__algorithm/pstl_merge.h | 2 +- libcxx/include/__algorithm/pstl_move.h | 2 +- libcxx/include/__algorithm/pstl_replace.h | 2 +- libcxx/include/__algorithm/pstl_rotate_copy.h | 2 +- libcxx/include/__algorithm/pstl_sort.h | 2 +- libcxx/include/__algorithm/pstl_stable_sort.h | 2 +- libcxx/include/__algorithm/pstl_transform.h | 2 +- libcxx/include/__config_site.in | 6 +-- .../include/__numeric/pstl_transform_reduce.h | 2 +- .../backends}/libdispatch.h | 19 ++++++-- .../cpu_backends => __pstl/backends}/serial.h | 19 ++++++-- .../thread.h => __pstl/backends/std_thread.h} | 19 ++++++-- libcxx/include/__pstl/configuration.h | 27 +++++++++++ .../configuration_fwd.h} | 25 +++++++--- libcxx/include/libcxx.imp | 6 --- libcxx/include/module.modulemap | 47 ++++++++----------- libcxx/src/CMakeLists.txt | 2 +- libcxx/src/pstl/libdispatch.cpp | 2 +- ...pstl.libdispatch.chunk_partitions.pass.cpp | 4 +- .../apple/system-install-properties.sh.cpp | 2 +- libcxx/utils/libcxx/test/features.py | 2 +- 40 files changed, 154 insertions(+), 169 deletions(-) delete mode 100644 libcxx/include/__algorithm/pstl_backends/cpu_backend.h delete mode 100644 libcxx/include/__algorithm/pstl_backends/cpu_backends/backend.h rename libcxx/include/{__algorithm/pstl_backends/cpu_backends => __pstl/backends}/libdispatch.h (94%) rename libcxx/include/{__algorithm/pstl_backends/cpu_backends => __pstl/backends}/serial.h (78%) rename libcxx/include/{__algorithm/pstl_backends/cpu_backends/thread.h => __pstl/backends/std_thread.h} (79%) create mode 100644 libcxx/include/__pstl/configuration.h rename libcxx/include/{__algorithm/pstl_backend.h => __pstl/configuration_fwd.h} (93%) diff --git a/libcxx/CMakeLists.txt b/libcxx/CMakeLists.txt index 043d5a8295c1..2977c26646cb 100644 --- a/libcxx/CMakeLists.txt +++ b/libcxx/CMakeLists.txt @@ -300,9 +300,9 @@ option(LIBCXX_HAS_EXTERNAL_THREAD_API This option may only be set to ON when LIBCXX_ENABLE_THREADS=ON." OFF) if (LIBCXX_ENABLE_THREADS) - set(LIBCXX_PSTL_CPU_BACKEND "std_thread" CACHE STRING "Which PSTL CPU backend to use") + set(LIBCXX_PSTL_BACKEND "std_thread" CACHE STRING "Which PSTL backend to use") else() - set(LIBCXX_PSTL_CPU_BACKEND "serial" CACHE STRING "Which PSTL CPU backend to use") + set(LIBCXX_PSTL_BACKEND "serial" CACHE STRING "Which PSTL backend to use") endif() # Misc options ---------------------------------------------------------------- @@ -792,14 +792,14 @@ elseif (LIBCXX_HARDENING_MODE STREQUAL "debug") config_define(8 _LIBCPP_HARDENING_MODE_DEFAULT) endif() -if (LIBCXX_PSTL_CPU_BACKEND STREQUAL "serial") - config_define(1 _LIBCPP_PSTL_CPU_BACKEND_SERIAL) -elseif(LIBCXX_PSTL_CPU_BACKEND STREQUAL "std_thread") - config_define(1 _LIBCPP_PSTL_CPU_BACKEND_THREAD) -elseif(LIBCXX_PSTL_CPU_BACKEND STREQUAL "libdispatch") - config_define(1 _LIBCPP_PSTL_CPU_BACKEND_LIBDISPATCH) +if (LIBCXX_PSTL_BACKEND STREQUAL "serial") + config_define(1 _LIBCPP_PSTL_BACKEND_SERIAL) +elseif(LIBCXX_PSTL_BACKEND STREQUAL "std_thread") + config_define(1 _LIBCPP_PSTL_BACKEND_STD_THREAD) +elseif(LIBCXX_PSTL_BACKEND STREQUAL "libdispatch") + config_define(1 _LIBCPP_PSTL_BACKEND_LIBDISPATCH) else() - message(FATAL_ERROR "LIBCXX_PSTL_CPU_BACKEND is set to ${LIBCXX_PSTL_CPU_BACKEND}, which is not a valid backend. + message(FATAL_ERROR "LIBCXX_PSTL_BACKEND is set to ${LIBCXX_PSTL_BACKEND}, which is not a valid backend. Valid backends are: serial, std_thread and libdispatch") endif() diff --git a/libcxx/cmake/caches/Apple.cmake b/libcxx/cmake/caches/Apple.cmake index cec13c08acf1..8768653e620a 100644 --- a/libcxx/cmake/caches/Apple.cmake +++ b/libcxx/cmake/caches/Apple.cmake @@ -7,7 +7,7 @@ set(LIBCXX_ENABLE_STATIC ON CACHE BOOL "") set(LIBCXX_ENABLE_SHARED ON CACHE BOOL "") set(LIBCXX_CXX_ABI libcxxabi CACHE STRING "") set(LIBCXX_ENABLE_VENDOR_AVAILABILITY_ANNOTATIONS ON CACHE BOOL "") -set(LIBCXX_PSTL_CPU_BACKEND libdispatch CACHE STRING "") +set(LIBCXX_PSTL_BACKEND libdispatch CACHE STRING "") set(LIBCXX_HERMETIC_STATIC_LIBRARY ON CACHE BOOL "") set(LIBCXXABI_HERMETIC_STATIC_LIBRARY ON CACHE BOOL "") diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt index a2af1d9915be..ee4979bfc6f8 100644 --- a/libcxx/include/CMakeLists.txt +++ b/libcxx/include/CMakeLists.txt @@ -73,18 +73,12 @@ set(files __algorithm/pop_heap.h __algorithm/prev_permutation.h __algorithm/pstl_any_all_none_of.h - __algorithm/pstl_backend.h - __algorithm/pstl_backends/cpu_backend.h __algorithm/pstl_backends/cpu_backends/any_of.h - __algorithm/pstl_backends/cpu_backends/backend.h __algorithm/pstl_backends/cpu_backends/fill.h __algorithm/pstl_backends/cpu_backends/find_if.h __algorithm/pstl_backends/cpu_backends/for_each.h - __algorithm/pstl_backends/cpu_backends/libdispatch.h __algorithm/pstl_backends/cpu_backends/merge.h - __algorithm/pstl_backends/cpu_backends/serial.h __algorithm/pstl_backends/cpu_backends/stable_sort.h - __algorithm/pstl_backends/cpu_backends/thread.h __algorithm/pstl_backends/cpu_backends/transform.h __algorithm/pstl_backends/cpu_backends/transform_reduce.h __algorithm/pstl_copy.h @@ -594,6 +588,11 @@ set(files __numeric/transform_exclusive_scan.h __numeric/transform_inclusive_scan.h __numeric/transform_reduce.h + __pstl/backends/libdispatch.h + __pstl/backends/serial.h + __pstl/backends/std_thread.h + __pstl/configuration.h + __pstl/configuration_fwd.h __pstl/cpu_algos/cpu_traits.h __random/bernoulli_distribution.h __random/binomial_distribution.h diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backend.h b/libcxx/include/__algorithm/pstl_backends/cpu_backend.h deleted file mode 100644 index 53eae58f9609..000000000000 --- a/libcxx/include/__algorithm/pstl_backends/cpu_backend.h +++ /dev/null @@ -1,23 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKEND_H -#define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKEND_H - -#include <__algorithm/pstl_backends/cpu_backends/any_of.h> -#include <__algorithm/pstl_backends/cpu_backends/backend.h> -#include <__algorithm/pstl_backends/cpu_backends/fill.h> -#include <__algorithm/pstl_backends/cpu_backends/find_if.h> -#include <__algorithm/pstl_backends/cpu_backends/for_each.h> -#include <__algorithm/pstl_backends/cpu_backends/merge.h> -#include <__algorithm/pstl_backends/cpu_backends/stable_sort.h> -#include <__algorithm/pstl_backends/cpu_backends/transform.h> -#include <__algorithm/pstl_backends/cpu_backends/transform_reduce.h> -#include <__config> - -#endif // _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKEND_H diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/any_of.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/any_of.h index 3755d288047e..3db4765da64b 100644 --- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/any_of.h +++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/any_of.h @@ -11,12 +11,12 @@ #include <__algorithm/any_of.h> #include <__algorithm/find_if.h> -#include <__algorithm/pstl_backends/cpu_backends/backend.h> #include <__atomic/atomic.h> #include <__atomic/memory_order.h> #include <__config> #include <__functional/operations.h> #include <__iterator/concepts.h> +#include <__pstl/configuration_fwd.h> #include <__pstl/cpu_algos/cpu_traits.h> #include <__type_traits/is_execution_policy.h> #include <__utility/move.h> diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/backend.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/backend.h deleted file mode 100644 index cb9425862a2b..000000000000 --- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/backend.h +++ /dev/null @@ -1,45 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKEND_BACKEND_H -#define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKEND_BACKEND_H - -#include <__config> -#include - -#if defined(_LIBCPP_PSTL_CPU_BACKEND_SERIAL) -# include <__algorithm/pstl_backends/cpu_backends/serial.h> -#elif defined(_LIBCPP_PSTL_CPU_BACKEND_THREAD) -# include <__algorithm/pstl_backends/cpu_backends/thread.h> -#elif defined(_LIBCPP_PSTL_CPU_BACKEND_LIBDISPATCH) -# include <__algorithm/pstl_backends/cpu_backends/libdispatch.h> -#else -# error "Invalid CPU backend choice" -#endif - -#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) -# pragma GCC system_header -#endif - -#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17 - -_LIBCPP_BEGIN_NAMESPACE_STD - -# if defined(_LIBCPP_PSTL_CPU_BACKEND_SERIAL) -using __cpu_backend_tag = __pstl::__serial_backend_tag; -# elif defined(_LIBCPP_PSTL_CPU_BACKEND_THREAD) -using __cpu_backend_tag = __pstl::__std_thread_backend_tag; -# elif defined(_LIBCPP_PSTL_CPU_BACKEND_LIBDISPATCH) -using __cpu_backend_tag = __pstl::__libdispatch_backend_tag; -# endif - -_LIBCPP_END_NAMESPACE_STD - -#endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && && _LIBCPP_STD_VER >= 17 - -#endif // _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKEND_BACKEND_H diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/fill.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/fill.h index 0c20bdff6267..b5a49f8417d3 100644 --- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/fill.h +++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/fill.h @@ -10,9 +10,9 @@ #define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_FILL_H #include <__algorithm/fill.h> -#include <__algorithm/pstl_backends/cpu_backends/backend.h> #include <__config> #include <__iterator/concepts.h> +#include <__pstl/configuration_fwd.h> #include <__pstl/cpu_algos/cpu_traits.h> #include <__type_traits/is_execution_policy.h> #include <__utility/empty.h> diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/find_if.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/find_if.h index 626293faef69..2b1754ea3a75 100644 --- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/find_if.h +++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/find_if.h @@ -10,12 +10,12 @@ #define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_FIND_IF_H #include <__algorithm/find_if.h> -#include <__algorithm/pstl_backends/cpu_backends/backend.h> #include <__atomic/atomic.h> #include <__config> #include <__functional/operations.h> #include <__iterator/concepts.h> #include <__iterator/iterator_traits.h> +#include <__pstl/configuration_fwd.h> #include <__pstl/cpu_algos/cpu_traits.h> #include <__type_traits/is_execution_policy.h> #include <__utility/move.h> diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/for_each.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/for_each.h index d637084e151d..6db212ead8ae 100644 --- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/for_each.h +++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/for_each.h @@ -10,9 +10,9 @@ #define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKNEDS_FOR_EACH_H #include <__algorithm/for_each.h> -#include <__algorithm/pstl_backends/cpu_backends/backend.h> #include <__config> #include <__iterator/concepts.h> +#include <__pstl/configuration_fwd.h> #include <__pstl/cpu_algos/cpu_traits.h> #include <__type_traits/is_execution_policy.h> #include <__utility/empty.h> diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/merge.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/merge.h index c93f4051c9d0..f3e59e8c0285 100644 --- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/merge.h +++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/merge.h @@ -10,9 +10,9 @@ #define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_MERGE_H #include <__algorithm/merge.h> -#include <__algorithm/pstl_backends/cpu_backends/backend.h> #include <__config> #include <__iterator/concepts.h> +#include <__pstl/configuration_fwd.h> #include <__pstl/cpu_algos/cpu_traits.h> #include <__type_traits/is_execution_policy.h> #include <__utility/move.h> diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/stable_sort.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/stable_sort.h index 8c60cf897ff8..9ad8cc8fb0f2 100644 --- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/stable_sort.h +++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/stable_sort.h @@ -9,9 +9,9 @@ #ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_STABLE_SORT_H #define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_STABLE_SORT_H -#include <__algorithm/pstl_backends/cpu_backends/backend.h> #include <__algorithm/stable_sort.h> #include <__config> +#include <__pstl/configuration_fwd.h> #include <__pstl/cpu_algos/cpu_traits.h> #include <__type_traits/is_execution_policy.h> #include <__utility/empty.h> diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform.h index 4b9b29686683..65e166d847e1 100644 --- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform.h +++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform.h @@ -9,11 +9,11 @@ #ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_TRANSFORM_H #define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_TRANSFORM_H -#include <__algorithm/pstl_backends/cpu_backends/backend.h> #include <__algorithm/transform.h> #include <__config> #include <__iterator/concepts.h> #include <__iterator/iterator_traits.h> +#include <__pstl/configuration_fwd.h> #include <__pstl/cpu_algos/cpu_traits.h> #include <__type_traits/enable_if.h> #include <__type_traits/is_execution_policy.h> diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform_reduce.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform_reduce.h index c074eea9861c..af481d505bb9 100644 --- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform_reduce.h +++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform_reduce.h @@ -9,11 +9,11 @@ #ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_TRANSFORM_REDUCE_H #define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_TRANSFORM_REDUCE_H -#include <__algorithm/pstl_backends/cpu_backends/backend.h> #include <__config> #include <__iterator/concepts.h> #include <__iterator/iterator_traits.h> #include <__numeric/transform_reduce.h> +#include <__pstl/configuration_fwd.h> #include <__pstl/cpu_algos/cpu_traits.h> #include <__type_traits/desugars_to.h> #include <__type_traits/is_arithmetic.h> diff --git a/libcxx/include/__algorithm/pstl_copy.h b/libcxx/include/__algorithm/pstl_copy.h index f35bb9713ef1..0fcea33c3919 100644 --- a/libcxx/include/__algorithm/pstl_copy.h +++ b/libcxx/include/__algorithm/pstl_copy.h @@ -10,13 +10,13 @@ #define _LIBCPP___ALGORITHM_PSTL_COPY_H #include <__algorithm/copy_n.h> -#include <__algorithm/pstl_backend.h> #include <__algorithm/pstl_frontend_dispatch.h> #include <__algorithm/pstl_transform.h> #include <__config> #include <__functional/identity.h> #include <__iterator/concepts.h> #include <__iterator/cpp17_iterator_concepts.h> +#include <__pstl/configuration.h> #include <__type_traits/enable_if.h> #include <__type_traits/is_constant_evaluated.h> #include <__type_traits/is_execution_policy.h> diff --git a/libcxx/include/__algorithm/pstl_count.h b/libcxx/include/__algorithm/pstl_count.h index 6ff57cac334e..64c84d855e4f 100644 --- a/libcxx/include/__algorithm/pstl_count.h +++ b/libcxx/include/__algorithm/pstl_count.h @@ -11,7 +11,6 @@ #include <__algorithm/count.h> #include <__algorithm/for_each.h> -#include <__algorithm/pstl_backend.h> #include <__algorithm/pstl_for_each.h> #include <__algorithm/pstl_frontend_dispatch.h> #include <__atomic/atomic.h> @@ -20,6 +19,7 @@ #include <__iterator/cpp17_iterator_concepts.h> #include <__iterator/iterator_traits.h> #include <__numeric/pstl_transform_reduce.h> +#include <__pstl/configuration.h> #include <__type_traits/enable_if.h> #include <__type_traits/is_execution_policy.h> #include <__type_traits/remove_cvref.h> diff --git a/libcxx/include/__algorithm/pstl_find.h b/libcxx/include/__algorithm/pstl_find.h index 3b30a7bc9b45..b4c4dfb2ffb6 100644 --- a/libcxx/include/__algorithm/pstl_find.h +++ b/libcxx/include/__algorithm/pstl_find.h @@ -11,10 +11,10 @@ #include <__algorithm/comp.h> #include <__algorithm/find.h> -#include <__algorithm/pstl_backend.h> #include <__algorithm/pstl_frontend_dispatch.h> #include <__config> #include <__iterator/cpp17_iterator_concepts.h> +#include <__pstl/configuration.h> #include <__type_traits/enable_if.h> #include <__type_traits/is_execution_policy.h> #include <__type_traits/remove_cvref.h> diff --git a/libcxx/include/__algorithm/pstl_for_each.h b/libcxx/include/__algorithm/pstl_for_each.h index a9ebed74a62f..a99eb6d97fd2 100644 --- a/libcxx/include/__algorithm/pstl_for_each.h +++ b/libcxx/include/__algorithm/pstl_for_each.h @@ -11,11 +11,11 @@ #include <__algorithm/for_each.h> #include <__algorithm/for_each_n.h> -#include <__algorithm/pstl_backend.h> #include <__algorithm/pstl_frontend_dispatch.h> #include <__config> #include <__iterator/concepts.h> #include <__iterator/cpp17_iterator_concepts.h> +#include <__pstl/configuration.h> #include <__type_traits/enable_if.h> #include <__type_traits/is_execution_policy.h> #include <__type_traits/remove_cvref.h> diff --git a/libcxx/include/__algorithm/pstl_generate.h b/libcxx/include/__algorithm/pstl_generate.h index 886af290d7f2..350c0e4798be 100644 --- a/libcxx/include/__algorithm/pstl_generate.h +++ b/libcxx/include/__algorithm/pstl_generate.h @@ -9,12 +9,12 @@ #ifndef _LIBCPP___ALGORITHM_PSTL_GENERATE_H #define _LIBCPP___ALGORITHM_PSTL_GENERATE_H -#include <__algorithm/pstl_backend.h> #include <__algorithm/pstl_for_each.h> #include <__algorithm/pstl_frontend_dispatch.h> #include <__config> #include <__iterator/cpp17_iterator_concepts.h> #include <__iterator/iterator_traits.h> +#include <__pstl/configuration.h> #include <__type_traits/enable_if.h> #include <__type_traits/is_execution_policy.h> #include <__type_traits/remove_cvref.h> diff --git a/libcxx/include/__algorithm/pstl_is_partitioned.h b/libcxx/include/__algorithm/pstl_is_partitioned.h index 108bb1e43252..c016b388e378 100644 --- a/libcxx/include/__algorithm/pstl_is_partitioned.h +++ b/libcxx/include/__algorithm/pstl_is_partitioned.h @@ -10,11 +10,11 @@ #define _LIBCPP___ALGORITHM_PSTL_IS_PARITTIONED #include <__algorithm/pstl_any_all_none_of.h> -#include <__algorithm/pstl_backend.h> #include <__algorithm/pstl_find.h> #include <__algorithm/pstl_frontend_dispatch.h> #include <__config> #include <__iterator/cpp17_iterator_concepts.h> +#include <__pstl/configuration.h> #include <__type_traits/enable_if.h> #include <__type_traits/is_execution_policy.h> #include <__type_traits/remove_cvref.h> diff --git a/libcxx/include/__algorithm/pstl_merge.h b/libcxx/include/__algorithm/pstl_merge.h index d03cd8c7fbd5..87f634a67f58 100644 --- a/libcxx/include/__algorithm/pstl_merge.h +++ b/libcxx/include/__algorithm/pstl_merge.h @@ -9,10 +9,10 @@ #ifndef _LIBCPP___ALGORITHM_PSTL_MERGE_H #define _LIBCPP___ALGORITHM_PSTL_MERGE_H -#include <__algorithm/pstl_backend.h> #include <__config> #include <__functional/operations.h> #include <__iterator/cpp17_iterator_concepts.h> +#include <__pstl/configuration.h> #include <__type_traits/enable_if.h> #include <__type_traits/is_execution_policy.h> #include <__type_traits/remove_cvref.h> diff --git a/libcxx/include/__algorithm/pstl_move.h b/libcxx/include/__algorithm/pstl_move.h index f4c8c1fbb2e8..3155ddedf91b 100644 --- a/libcxx/include/__algorithm/pstl_move.h +++ b/libcxx/include/__algorithm/pstl_move.h @@ -10,13 +10,13 @@ #define _LIBCPP___ALGORITHM_PSTL_MOVE_H #include <__algorithm/copy_n.h> -#include <__algorithm/pstl_backend.h> #include <__algorithm/pstl_frontend_dispatch.h> #include <__algorithm/pstl_transform.h> #include <__config> #include <__functional/identity.h> #include <__iterator/cpp17_iterator_concepts.h> #include <__iterator/iterator_traits.h> +#include <__pstl/configuration.h> #include <__type_traits/enable_if.h> #include <__type_traits/is_constant_evaluated.h> #include <__type_traits/is_execution_policy.h> diff --git a/libcxx/include/__algorithm/pstl_replace.h b/libcxx/include/__algorithm/pstl_replace.h index 73ac11cda26a..b2ded54dfe25 100644 --- a/libcxx/include/__algorithm/pstl_replace.h +++ b/libcxx/include/__algorithm/pstl_replace.h @@ -9,13 +9,13 @@ #ifndef _LIBCPP___ALGORITHM_PSTL_REPLACE_H #define _LIBCPP___ALGORITHM_PSTL_REPLACE_H -#include <__algorithm/pstl_backend.h> #include <__algorithm/pstl_for_each.h> #include <__algorithm/pstl_frontend_dispatch.h> #include <__algorithm/pstl_transform.h> #include <__config> #include <__iterator/cpp17_iterator_concepts.h> #include <__iterator/iterator_traits.h> +#include <__pstl/configuration.h> #include <__type_traits/enable_if.h> #include <__type_traits/remove_cvref.h> #include <__utility/move.h> diff --git a/libcxx/include/__algorithm/pstl_rotate_copy.h b/libcxx/include/__algorithm/pstl_rotate_copy.h index adab3958fe31..1a32b710877c 100644 --- a/libcxx/include/__algorithm/pstl_rotate_copy.h +++ b/libcxx/include/__algorithm/pstl_rotate_copy.h @@ -9,10 +9,10 @@ #ifndef _LIBCPP___ALGORITHM_PSTL_ROTATE_COPY_H #define _LIBCPP___ALGORITHM_PSTL_ROTATE_COPY_H -#include <__algorithm/pstl_backend.h> #include <__algorithm/pstl_copy.h> #include <__algorithm/pstl_frontend_dispatch.h> #include <__iterator/cpp17_iterator_concepts.h> +#include <__pstl/configuration.h> #include <__type_traits/is_execution_policy.h> #include diff --git a/libcxx/include/__algorithm/pstl_sort.h b/libcxx/include/__algorithm/pstl_sort.h index 65bc794ca6f4..769dd81af77e 100644 --- a/libcxx/include/__algorithm/pstl_sort.h +++ b/libcxx/include/__algorithm/pstl_sort.h @@ -9,12 +9,12 @@ #ifndef _LIBCPP___ALGORITHM_PSTL_SORT_H #define _LIBCPP___ALGORITHM_PSTL_SORT_H -#include <__algorithm/pstl_backend.h> #include <__algorithm/pstl_frontend_dispatch.h> #include <__algorithm/pstl_stable_sort.h> #include <__config> #include <__functional/operations.h> #include <__iterator/cpp17_iterator_concepts.h> +#include <__pstl/configuration.h> #include <__type_traits/is_execution_policy.h> #include <__type_traits/remove_cvref.h> #include <__utility/empty.h> diff --git a/libcxx/include/__algorithm/pstl_stable_sort.h b/libcxx/include/__algorithm/pstl_stable_sort.h index 79b94557e3dc..f5e0dd40f72b 100644 --- a/libcxx/include/__algorithm/pstl_stable_sort.h +++ b/libcxx/include/__algorithm/pstl_stable_sort.h @@ -9,10 +9,10 @@ #ifndef _LIBCPP___ALGORITHM_PSTL_STABLE_SORT_H #define _LIBCPP___ALGORITHM_PSTL_STABLE_SORT_H -#include <__algorithm/pstl_backend.h> #include <__config> #include <__functional/operations.h> #include <__iterator/cpp17_iterator_concepts.h> +#include <__pstl/configuration.h> #include <__type_traits/enable_if.h> #include <__type_traits/is_execution_policy.h> #include <__type_traits/remove_cvref.h> diff --git a/libcxx/include/__algorithm/pstl_transform.h b/libcxx/include/__algorithm/pstl_transform.h index a01a64a43cf1..80e1d6b496f2 100644 --- a/libcxx/include/__algorithm/pstl_transform.h +++ b/libcxx/include/__algorithm/pstl_transform.h @@ -9,9 +9,9 @@ #ifndef _LIBCPP___ALGORITHM_PSTL_TRANSFORM_H #define _LIBCPP___ALGORITHM_PSTL_TRANSFORM_H -#include <__algorithm/pstl_backend.h> #include <__config> #include <__iterator/cpp17_iterator_concepts.h> +#include <__pstl/configuration.h> #include <__type_traits/enable_if.h> #include <__type_traits/is_execution_policy.h> #include <__type_traits/remove_cvref.h> diff --git a/libcxx/include/__config_site.in b/libcxx/include/__config_site.in index 7c002c5bfcf8..89a14609ee3f 100644 --- a/libcxx/include/__config_site.in +++ b/libcxx/include/__config_site.in @@ -32,9 +32,9 @@ #cmakedefine _LIBCPP_INSTRUMENTED_WITH_ASAN // PSTL backends -#cmakedefine _LIBCPP_PSTL_CPU_BACKEND_SERIAL -#cmakedefine _LIBCPP_PSTL_CPU_BACKEND_THREAD -#cmakedefine _LIBCPP_PSTL_CPU_BACKEND_LIBDISPATCH +#cmakedefine _LIBCPP_PSTL_BACKEND_SERIAL +#cmakedefine _LIBCPP_PSTL_BACKEND_STD_THREAD +#cmakedefine _LIBCPP_PSTL_BACKEND_LIBDISPATCH // Hardening. #cmakedefine _LIBCPP_HARDENING_MODE_DEFAULT @_LIBCPP_HARDENING_MODE_DEFAULT@ diff --git a/libcxx/include/__numeric/pstl_transform_reduce.h b/libcxx/include/__numeric/pstl_transform_reduce.h index 2d2621dc8dad..fe41b1c86f3b 100644 --- a/libcxx/include/__numeric/pstl_transform_reduce.h +++ b/libcxx/include/__numeric/pstl_transform_reduce.h @@ -9,12 +9,12 @@ #ifndef _LIBCPP___NUMERIC_PSTL_TRANSFORM_REDUCE_H #define _LIBCPP___NUMERIC_PSTL_TRANSFORM_REDUCE_H -#include <__algorithm/pstl_backend.h> #include <__algorithm/pstl_frontend_dispatch.h> #include <__config> #include <__functional/operations.h> #include <__iterator/cpp17_iterator_concepts.h> #include <__numeric/transform_reduce.h> +#include <__pstl/configuration.h> #include <__type_traits/is_execution_policy.h> #include <__utility/move.h> #include diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/libdispatch.h b/libcxx/include/__pstl/backends/libdispatch.h similarity index 94% rename from libcxx/include/__algorithm/pstl_backends/cpu_backends/libdispatch.h rename to libcxx/include/__pstl/backends/libdispatch.h index 17faadf55dd4..977b06b9a489 100644 --- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/libdispatch.h +++ b/libcxx/include/__pstl/backends/libdispatch.h @@ -6,8 +6,8 @@ // //===----------------------------------------------------------------------===// -#ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_LIBDISPATCH_H -#define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_LIBDISPATCH_H +#ifndef _LIBCPP___PSTL_BACKENDS_LIBDISPATCH_H +#define _LIBCPP___PSTL_BACKENDS_LIBDISPATCH_H #include <__algorithm/inplace_merge.h> #include <__algorithm/lower_bound.h> @@ -23,6 +23,7 @@ #include <__memory/construct_at.h> #include <__memory/unique_ptr.h> #include <__numeric/reduce.h> +#include <__pstl/configuration_fwd.h> #include <__pstl/cpu_algos/cpu_traits.h> #include <__utility/empty.h> #include <__utility/exception_guard.h> @@ -40,8 +41,6 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD namespace __pstl { -struct __libdispatch_backend_tag {}; - namespace __libdispatch { // ::dispatch_apply is marked as __attribute__((nothrow)) because it doesn't let exceptions propagate, and neither do // we. @@ -349,4 +348,14 @@ _LIBCPP_END_NAMESPACE_STD _LIBCPP_POP_MACROS -#endif // _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_LIBDISPATCH_H +// Implement PSTL algorithms based on the __cpu_traits specialized above +#include <__algorithm/pstl_backends/cpu_backends/any_of.h> +#include <__algorithm/pstl_backends/cpu_backends/fill.h> +#include <__algorithm/pstl_backends/cpu_backends/find_if.h> +#include <__algorithm/pstl_backends/cpu_backends/for_each.h> +#include <__algorithm/pstl_backends/cpu_backends/merge.h> +#include <__algorithm/pstl_backends/cpu_backends/stable_sort.h> +#include <__algorithm/pstl_backends/cpu_backends/transform.h> +#include <__algorithm/pstl_backends/cpu_backends/transform_reduce.h> + +#endif // _LIBCPP___PSTL_BACKENDS_LIBDISPATCH_H diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/serial.h b/libcxx/include/__pstl/backends/serial.h similarity index 78% rename from libcxx/include/__algorithm/pstl_backends/cpu_backends/serial.h rename to libcxx/include/__pstl/backends/serial.h index 7544619a8eef..8bb894509309 100644 --- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/serial.h +++ b/libcxx/include/__pstl/backends/serial.h @@ -7,10 +7,11 @@ // //===----------------------------------------------------------------------===// -#ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_SERIAL_H -#define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_SERIAL_H +#ifndef _LIBCPP___PSTL_BACKENDS_SERIAL_H +#define _LIBCPP___PSTL_BACKENDS_SERIAL_H #include <__config> +#include <__pstl/configuration_fwd.h> #include <__pstl/cpu_algos/cpu_traits.h> #include <__utility/empty.h> #include <__utility/move.h> @@ -29,8 +30,6 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD namespace __pstl { -struct __serial_backend_tag {}; - template <> struct __cpu_traits<__serial_backend_tag> { template @@ -82,4 +81,14 @@ _LIBCPP_POP_MACROS #endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && && _LIBCPP_STD_VER >= 17 -#endif // _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_SERIAL_H +// Implement PSTL algorithms based on the __cpu_traits specialized above +#include <__algorithm/pstl_backends/cpu_backends/any_of.h> +#include <__algorithm/pstl_backends/cpu_backends/fill.h> +#include <__algorithm/pstl_backends/cpu_backends/find_if.h> +#include <__algorithm/pstl_backends/cpu_backends/for_each.h> +#include <__algorithm/pstl_backends/cpu_backends/merge.h> +#include <__algorithm/pstl_backends/cpu_backends/stable_sort.h> +#include <__algorithm/pstl_backends/cpu_backends/transform.h> +#include <__algorithm/pstl_backends/cpu_backends/transform_reduce.h> + +#endif // _LIBCPP___PSTL_BACKENDS_SERIAL_H diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/thread.h b/libcxx/include/__pstl/backends/std_thread.h similarity index 79% rename from libcxx/include/__algorithm/pstl_backends/cpu_backends/thread.h rename to libcxx/include/__pstl/backends/std_thread.h index 2acf912264a0..ab09f42cfdd8 100644 --- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/thread.h +++ b/libcxx/include/__pstl/backends/std_thread.h @@ -6,11 +6,12 @@ // //===----------------------------------------------------------------------===// -#ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_THREAD_H -#define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_THREAD_H +#ifndef _LIBCPP___PSTL_BACKENDS_STD_THREAD_H +#define _LIBCPP___PSTL_BACKENDS_STD_THREAD_H #include <__assert> #include <__config> +#include <__pstl/configuration_fwd.h> #include <__pstl/cpu_algos/cpu_traits.h> #include <__utility/empty.h> #include <__utility/move.h> @@ -32,8 +33,6 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD namespace __pstl { -struct __std_thread_backend_tag {}; - template <> struct __cpu_traits<__std_thread_backend_tag> { template @@ -85,4 +84,14 @@ _LIBCPP_END_NAMESPACE_STD _LIBCPP_POP_MACROS -#endif // _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_THREAD_H +// Implement PSTL algorithms based on the __cpu_traits specialized above +#include <__algorithm/pstl_backends/cpu_backends/any_of.h> +#include <__algorithm/pstl_backends/cpu_backends/fill.h> +#include <__algorithm/pstl_backends/cpu_backends/find_if.h> +#include <__algorithm/pstl_backends/cpu_backends/for_each.h> +#include <__algorithm/pstl_backends/cpu_backends/merge.h> +#include <__algorithm/pstl_backends/cpu_backends/stable_sort.h> +#include <__algorithm/pstl_backends/cpu_backends/transform.h> +#include <__algorithm/pstl_backends/cpu_backends/transform_reduce.h> + +#endif // _LIBCPP___PSTL_BACKENDS_STD_THREAD_H diff --git a/libcxx/include/__pstl/configuration.h b/libcxx/include/__pstl/configuration.h new file mode 100644 index 000000000000..d32bd21df1f9 --- /dev/null +++ b/libcxx/include/__pstl/configuration.h @@ -0,0 +1,27 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___PSTL_CONFIGURATION_H +#define _LIBCPP___PSTL_CONFIGURATION_H + +#include <__config> +#include <__pstl/configuration_fwd.h> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +#if defined(_LIBCPP_PSTL_BACKEND_SERIAL) +# include <__pstl/backends/serial.h> +#elif defined(_LIBCPP_PSTL_BACKEND_STD_THREAD) +# include <__pstl/backends/std_thread.h> +#elif defined(_LIBCPP_PSTL_BACKEND_LIBDISPATCH) +# include <__pstl/backends/libdispatch.h> +#endif + +#endif // _LIBCPP___PSTL_CONFIGURATION_H diff --git a/libcxx/include/__algorithm/pstl_backend.h b/libcxx/include/__pstl/configuration_fwd.h similarity index 93% rename from libcxx/include/__algorithm/pstl_backend.h rename to libcxx/include/__pstl/configuration_fwd.h index 3af03ce2fbc8..995fcfce847c 100644 --- a/libcxx/include/__algorithm/pstl_backend.h +++ b/libcxx/include/__pstl/configuration_fwd.h @@ -6,10 +6,9 @@ // //===----------------------------------------------------------------------===// -#ifndef _LIBCPP___ALGORITHM_PSTL_BACKEND_H -#define _LIBCPP___ALGORITHM_PSTL_BACKEND_H +#ifndef _LIBCPP___PSTL_CONFIGURATION_FWD_H +#define _LIBCPP___PSTL_CONFIGURATION_FWD_H -#include <__algorithm/pstl_backends/cpu_backend.h> #include <__config> #include @@ -191,6 +190,20 @@ into a program termination at the front-end level. When a backend returns a dise frontend will turn that into a call to `std::__throw_bad_alloc();` to report the internal failure to the user. */ +namespace __pstl { +struct __libdispatch_backend_tag {}; +struct __serial_backend_tag {}; +struct __std_thread_backend_tag {}; +} // namespace __pstl + +# if defined(_LIBCPP_PSTL_BACKEND_SERIAL) +using __cpu_backend_tag = __pstl::__serial_backend_tag; +# elif defined(_LIBCPP_PSTL_BACKEND_STD_THREAD) +using __cpu_backend_tag = __pstl::__std_thread_backend_tag; +# elif defined(_LIBCPP_PSTL_BACKEND_LIBDISPATCH) +using __cpu_backend_tag = __pstl::__libdispatch_backend_tag; +# endif + template struct __select_backend; @@ -206,8 +219,8 @@ struct __select_backend { }; # endif -# if defined(_LIBCPP_PSTL_CPU_BACKEND_SERIAL) || defined(_LIBCPP_PSTL_CPU_BACKEND_THREAD) || \ - defined(_LIBCPP_PSTL_CPU_BACKEND_LIBDISPATCH) +# if defined(_LIBCPP_PSTL_BACKEND_SERIAL) || defined(_LIBCPP_PSTL_BACKEND_STD_THREAD) || \ + defined(_LIBCPP_PSTL_BACKEND_LIBDISPATCH) template <> struct __select_backend { using type = __cpu_backend_tag; @@ -229,4 +242,4 @@ _LIBCPP_END_NAMESPACE_STD #endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17 -#endif // _LIBCPP___ALGORITHM_PSTL_BACKEND_H +#endif // _LIBCPP___PSTL_CONFIGURATION_FWD_H diff --git a/libcxx/include/libcxx.imp b/libcxx/include/libcxx.imp index 8820fb8c0936..a4e2690fc55c 100644 --- a/libcxx/include/libcxx.imp +++ b/libcxx/include/libcxx.imp @@ -73,18 +73,12 @@ { include: [ "<__algorithm/pop_heap.h>", "private", "", "public" ] }, { include: [ "<__algorithm/prev_permutation.h>", "private", "", "public" ] }, { include: [ "<__algorithm/pstl_any_all_none_of.h>", "private", "", "public" ] }, - { include: [ "<__algorithm/pstl_backend.h>", "private", "", "public" ] }, - { include: [ "<__algorithm/pstl_backends/cpu_backend.h>", "private", "", "public" ] }, { include: [ "<__algorithm/pstl_backends/cpu_backends/any_of.h>", "private", "", "public" ] }, - { include: [ "<__algorithm/pstl_backends/cpu_backends/backend.h>", "private", "", "public" ] }, { include: [ "<__algorithm/pstl_backends/cpu_backends/fill.h>", "private", "", "public" ] }, { include: [ "<__algorithm/pstl_backends/cpu_backends/find_if.h>", "private", "", "public" ] }, { include: [ "<__algorithm/pstl_backends/cpu_backends/for_each.h>", "private", "", "public" ] }, - { include: [ "<__algorithm/pstl_backends/cpu_backends/libdispatch.h>", "private", "", "public" ] }, { include: [ "<__algorithm/pstl_backends/cpu_backends/merge.h>", "private", "", "public" ] }, - { include: [ "<__algorithm/pstl_backends/cpu_backends/serial.h>", "private", "", "public" ] }, { include: [ "<__algorithm/pstl_backends/cpu_backends/stable_sort.h>", "private", "", "public" ] }, - { include: [ "<__algorithm/pstl_backends/cpu_backends/thread.h>", "private", "", "public" ] }, { include: [ "<__algorithm/pstl_backends/cpu_backends/transform.h>", "private", "", "public" ] }, { include: [ "<__algorithm/pstl_backends/cpu_backends/transform_reduce.h>", "private", "", "public" ] }, { include: [ "<__algorithm/pstl_copy.h>", "private", "", "public" ] }, diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap index ce133e471deb..f996c2cc0545 100644 --- a/libcxx/include/module.modulemap +++ b/libcxx/include/module.modulemap @@ -714,32 +714,14 @@ module std_private_algorithm_partition_point [system module std_private_algorithm_pop_heap [system] { header "__algorithm/pop_heap.h" } module std_private_algorithm_prev_permutation [system] { header "__algorithm/prev_permutation.h" } module std_private_algorithm_pstl_any_all_none_of [system] { header "__algorithm/pstl_any_all_none_of.h" } -module std_private_algorithm_pstl_backend [system] { - header "__algorithm/pstl_backend.h" - export * -} -module std_private_algorithm_pstl_backends_cpu_backend [system] { - header "__algorithm/pstl_backends/cpu_backend.h" - export * -} -module std_private_algorithm_pstl_backends_cpu_backends_any_of [system] { header "__algorithm/pstl_backends/cpu_backends/any_of.h" } -module std_private_algorithm_pstl_backends_cpu_backends_backend [system] { - header "__algorithm/pstl_backends/cpu_backends/backend.h" - export * -} -module std_private_algorithm_pstl_backends_cpu_backends_fill [system] { header "__algorithm/pstl_backends/cpu_backends/fill.h" } -module std_private_algorithm_pstl_backends_cpu_backends_find_if [system] { header "__algorithm/pstl_backends/cpu_backends/find_if.h" } -module std_private_algorithm_pstl_backends_cpu_backends_for_each [system] { header "__algorithm/pstl_backends/cpu_backends/for_each.h" } -module std_private_algorithm_pstl_backends_cpu_backends_libdispatch [system] { header "__algorithm/pstl_backends/cpu_backends/libdispatch.h" } -module std_private_algorithm_pstl_backends_cpu_backends_merge [system] { header "__algorithm/pstl_backends/cpu_backends/merge.h" } -module std_private_algorithm_pstl_backends_cpu_backends_serial [system] { textual header "__algorithm/pstl_backends/cpu_backends/serial.h" } -module std_private_algorithm_pstl_backends_cpu_backends_stable_sort [system] { header "__algorithm/pstl_backends/cpu_backends/stable_sort.h" } -module std_private_algorithm_pstl_backends_cpu_backends_thread [system] { textual header "__algorithm/pstl_backends/cpu_backends/thread.h" } -module std_private_algorithm_pstl_backends_cpu_backends_transform [system] { - header "__algorithm/pstl_backends/cpu_backends/transform.h" - export std_private_algorithm_transform -} -module std_private_algorithm_pstl_backends_cpu_backends_transform_reduce [system] { header "__algorithm/pstl_backends/cpu_backends/transform_reduce.h" } +module std_private_algorithm_pstl_backends_cpu_backends_any_of [system] { textual header "__algorithm/pstl_backends/cpu_backends/any_of.h" } +module std_private_algorithm_pstl_backends_cpu_backends_fill [system] { textual header "__algorithm/pstl_backends/cpu_backends/fill.h" } +module std_private_algorithm_pstl_backends_cpu_backends_find_if [system] { textual header "__algorithm/pstl_backends/cpu_backends/find_if.h" } +module std_private_algorithm_pstl_backends_cpu_backends_for_each [system] { textual header "__algorithm/pstl_backends/cpu_backends/for_each.h" } +module std_private_algorithm_pstl_backends_cpu_backends_merge [system] { textual header "__algorithm/pstl_backends/cpu_backends/merge.h" } +module std_private_algorithm_pstl_backends_cpu_backends_stable_sort [system] { textual header "__algorithm/pstl_backends/cpu_backends/stable_sort.h" } +module std_private_algorithm_pstl_backends_cpu_backends_transform [system] { textual header "__algorithm/pstl_backends/cpu_backends/transform.h" } +module std_private_algorithm_pstl_backends_cpu_backends_transform_reduce [system] { textual header "__algorithm/pstl_backends/cpu_backends/transform_reduce.h" } module std_private_algorithm_pstl_copy [system] { header "__algorithm/pstl_copy.h" } module std_private_algorithm_pstl_count [system] { header "__algorithm/pstl_count.h" } module std_private_algorithm_pstl_equal [system] { header "__algorithm/pstl_equal.h" } @@ -1613,7 +1595,18 @@ module std_private_numeric_transform_exclusive_scan [system] { header "__numeric module std_private_numeric_transform_inclusive_scan [system] { header "__numeric/transform_inclusive_scan.h" } module std_private_numeric_transform_reduce [system] { header "__numeric/transform_reduce.h" } -module std_private_pstl_cpu_algos_cpu_traits [system] { header "__pstl/cpu_algos/cpu_traits.h" } +module std_private_pstl_backends_libdispatch [system] { header "__pstl/backends/libdispatch.h" } +module std_private_pstl_backends_serial [system] { header "__pstl/backends/serial.h" } +module std_private_pstl_backends_std_thread [system] { header "__pstl/backends/std_thread.h" } +module std_private_pstl_cpu_algos_cpu_traits [system] { header "__pstl/cpu_algos/cpu_traits.h" } +module std_private_pstl_configuration_fwd [system] { + header "__pstl/configuration_fwd.h" + export * +} +module std_private_pstl_configuration [system] { + header "__pstl/configuration.h" + export * +} module std_private_queue_fwd [system] { header "__fwd/queue.h" } diff --git a/libcxx/src/CMakeLists.txt b/libcxx/src/CMakeLists.txt index a4a3fee86457..8b28d1b89189 100644 --- a/libcxx/src/CMakeLists.txt +++ b/libcxx/src/CMakeLists.txt @@ -327,7 +327,7 @@ set(LIBCXX_EXPERIMENTAL_SOURCES experimental/keep.cpp ) -if (LIBCXX_PSTL_CPU_BACKEND STREQUAL "libdispatch") +if (LIBCXX_PSTL_BACKEND STREQUAL "libdispatch") list(APPEND LIBCXX_EXPERIMENTAL_SOURCES pstl/libdispatch.cpp ) diff --git a/libcxx/src/pstl/libdispatch.cpp b/libcxx/src/pstl/libdispatch.cpp index d997a9c73463..3dca702341c8 100644 --- a/libcxx/src/pstl/libdispatch.cpp +++ b/libcxx/src/pstl/libdispatch.cpp @@ -7,8 +7,8 @@ //===----------------------------------------------------------------------===// #include <__algorithm/min.h> -#include <__algorithm/pstl_backends/cpu_backends/libdispatch.h> #include <__config> +#include <__pstl/backends/libdispatch.h> #include _LIBCPP_BEGIN_NAMESPACE_STD diff --git a/libcxx/test/libcxx/algorithms/pstl.libdispatch.chunk_partitions.pass.cpp b/libcxx/test/libcxx/algorithms/pstl.libdispatch.chunk_partitions.pass.cpp index 8c7016a80b81..b48ac02dd79c 100644 --- a/libcxx/test/libcxx/algorithms/pstl.libdispatch.chunk_partitions.pass.cpp +++ b/libcxx/test/libcxx/algorithms/pstl.libdispatch.chunk_partitions.pass.cpp @@ -8,11 +8,11 @@ // -// REQUIRES: libcpp-pstl-cpu-backend-libdispatch +// REQUIRES: libcpp-pstl-backend-libdispatch // __chunk_partitions __partition_chunks(ptrdiff_t); -#include <__algorithm/pstl_backends/cpu_backends/libdispatch.h> +#include <__pstl/backends/libdispatch.h> #include #include diff --git a/libcxx/test/libcxx/vendor/apple/system-install-properties.sh.cpp b/libcxx/test/libcxx/vendor/apple/system-install-properties.sh.cpp index 3e2e080368f4..4ea27401e35d 100644 --- a/libcxx/test/libcxx/vendor/apple/system-install-properties.sh.cpp +++ b/libcxx/test/libcxx/vendor/apple/system-install-properties.sh.cpp @@ -45,4 +45,4 @@ // Make sure we use the libdispatch backend for the PSTL. // -// RUN: grep "%{include-dir}/__config_site" -e '#define _LIBCPP_PSTL_CPU_BACKEND_LIBDISPATCH' +// RUN: grep "%{include-dir}/__config_site" -e '#define _LIBCPP_PSTL_BACKEND_LIBDISPATCH' diff --git a/libcxx/utils/libcxx/test/features.py b/libcxx/utils/libcxx/test/features.py index 6ff16309546b..c81b56b1af54 100644 --- a/libcxx/utils/libcxx/test/features.py +++ b/libcxx/utils/libcxx/test/features.py @@ -318,7 +318,7 @@ macros = { "_LIBCPP_HAS_NO_WIDE_CHARACTERS": "no-wide-characters", "_LIBCPP_HAS_NO_TIME_ZONE_DATABASE": "no-tzdb", "_LIBCPP_HAS_NO_UNICODE": "libcpp-has-no-unicode", - "_LIBCPP_PSTL_CPU_BACKEND_LIBDISPATCH": "libcpp-pstl-cpu-backend-libdispatch", + "_LIBCPP_PSTL_BACKEND_LIBDISPATCH": "libcpp-pstl-backend-libdispatch", } for macro, feature in macros.items(): DEFAULT_FEATURES.append( -- GitLab From 8d49ce176414cd4d0d5d276fd721d9226e17e810 Mon Sep 17 00:00:00 2001 From: David Green Date: Wed, 17 Apr 2024 18:38:24 +0100 Subject: [PATCH 083/862] [GlobalISel][AArch64] Add LLRINT support (#88702) This hooks up G_INTRINSIC_LLRINT instructions, very similar to the lrint nodes that already exist. On AArch64 they are treated the same as lrint with the default return types. --- llvm/include/llvm/Support/TargetOpcodes.def | 3 + llvm/include/llvm/Target/GenericOpcodes.td | 6 ++ .../Target/GlobalISel/SelectionDAGCompat.td | 1 + llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 2 + .../CodeGen/GlobalISel/LegalizerHelper.cpp | 6 +- .../AArch64/GISel/AArch64LegalizerInfo.cpp | 2 +- .../AArch64/GISel/AArch64RegisterBankInfo.cpp | 1 + .../AArch64/GlobalISel/arm64-irtranslator.ll | 10 ++ .../AArch64/GlobalISel/legalize-llrint.mir | 98 +++++++++++++++++++ .../GlobalISel/legalizer-info-validation.mir | 4 + llvm/test/CodeGen/AArch64/llrint-conv-fp16.ll | 2 + llvm/test/CodeGen/AArch64/llrint-conv.ll | 58 +++++++---- .../builtins/match-table-replacerreg.td | 22 ++--- .../match-table-imms.td | 30 +++--- .../match-table-intrinsics.td | 16 +-- .../match-table-patfrag-root.td | 30 +++--- .../GlobalISelCombinerEmitter/match-table.td | 62 ++++++------ llvm/test/TableGen/GlobalISelEmitter.td | 2 +- 18 files changed, 251 insertions(+), 104 deletions(-) create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/legalize-llrint.mir diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def index 5765926d6d93..cb98f96af522 100644 --- a/llvm/include/llvm/Support/TargetOpcodes.def +++ b/llvm/include/llvm/Support/TargetOpcodes.def @@ -351,6 +351,9 @@ HANDLE_TARGET_OPCODE(G_INTRINSIC_ROUND) /// INTRINSIC round to integer intrinsic. HANDLE_TARGET_OPCODE(G_INTRINSIC_LRINT) +/// INTRINSIC long round to integer intrinsic. +HANDLE_TARGET_OPCODE(G_INTRINSIC_LLRINT) + /// INTRINSIC roundeven intrinsic. HANDLE_TARGET_OPCODE(G_INTRINSIC_ROUNDEVEN) diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td index d0f471eb29b6..e8cf8fcb647f 100644 --- a/llvm/include/llvm/Target/GenericOpcodes.td +++ b/llvm/include/llvm/Target/GenericOpcodes.td @@ -1089,6 +1089,12 @@ def G_INTRINSIC_LRINT : GenericInstruction { let hasSideEffects = false; } +def G_INTRINSIC_LLRINT : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type1:$src); + let hasSideEffects = false; +} + def G_INTRINSIC_ROUNDEVEN : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src1); diff --git a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td index dd4e7d790bc6..8fa0e4b86d6d 100644 --- a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td +++ b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td @@ -157,6 +157,7 @@ def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; +def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp index 9b4575f7f34d..0b6aae375975 100644 --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -1955,6 +1955,8 @@ unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) { return TargetOpcode::G_PTRMASK; case Intrinsic::lrint: return TargetOpcode::G_INTRINSIC_LRINT; + case Intrinsic::llrint: + return TargetOpcode::G_INTRINSIC_LLRINT; // FADD/FMUL require checking the FMF, so are handled elsewhere. case Intrinsic::vector_reduce_fmin: return TargetOpcode::G_VECREDUCE_FMIN; diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp index 156353296cfc..d55091e2e717 100644 --- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -474,6 +474,8 @@ static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) { RTLIBCASE(ROUNDEVEN_F); case TargetOpcode::G_INTRINSIC_LRINT: RTLIBCASE(LRINT_F); + case TargetOpcode::G_INTRINSIC_LLRINT: + RTLIBCASE(LLRINT_F); } llvm_unreachable("Unknown libcall function"); } @@ -1061,7 +1063,8 @@ LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) { return Status; break; } - case TargetOpcode::G_INTRINSIC_LRINT: { + case TargetOpcode::G_INTRINSIC_LRINT: + case TargetOpcode::G_INTRINSIC_LLRINT: { LLT LLTy = MRI.getType(MI.getOperand(1).getReg()); unsigned Size = LLTy.getSizeInBits(); Type *HLTy = getFloatTypeForLLT(Ctx, LLTy); @@ -2661,6 +2664,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) { case TargetOpcode::G_FPTOSI: case TargetOpcode::G_FPTOUI: case TargetOpcode::G_INTRINSIC_LRINT: + case TargetOpcode::G_INTRINSIC_LLRINT: case TargetOpcode::G_IS_FPCLASS: Observer.changingInstr(MI); diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp index 661ea151d1a0..85dd0f2eb192 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp @@ -262,7 +262,7 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) .minScalar(0, s32) .scalarize(0); - getActionDefinitionsBuilder(G_INTRINSIC_LRINT) + getActionDefinitionsBuilder({G_INTRINSIC_LRINT, G_INTRINSIC_LLRINT}) .legalFor({{s64, MinFPScalar}, {s64, s32}, {s64, s64}}) .libcallFor({{s64, s128}}) .minScalarOrElt(1, MinFPScalar); diff --git a/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp index d5c4ce1888e7..44ba9f0429e6 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp @@ -793,6 +793,7 @@ AArch64RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { case TargetOpcode::G_FPTOSI: case TargetOpcode::G_FPTOUI: case TargetOpcode::G_INTRINSIC_LRINT: + case TargetOpcode::G_INTRINSIC_LLRINT: if (MRI.getType(MI.getOperand(0).getReg()).isVector()) break; OpRegBankIdx = {PMI_FirstGPR, PMI_FirstFPR}; diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll index a131f35e66d0..a61931b898ae 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll +++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll @@ -1385,6 +1385,16 @@ define i32 @test_intrinsic_lrint(float %a) { ret i32 %res } +declare i32 @llvm.llrint.i32.f32(float) +define i32 @test_intrinsic_llrint(float %a) { +; CHECK-LABEL: name: test_intrinsic_llrint +; CHECK: [[A:%[0-9]+]]:_(s32) = COPY $s0 +; CHECK: [[RES:%[0-9]+]]:_(s32) = G_INTRINSIC_LLRINT [[A]] +; CHECK: $w0 = COPY [[RES]] + %res = call i32 @llvm.llrint.i32.f32(float %a) + ret i32 %res +} + declare i32 @llvm.ctlz.i32(i32, i1) define i32 @test_ctlz_intrinsic_zero_not_undef(i32 %a) { ; CHECK-LABEL: name: test_ctlz_intrinsic_zero_not_undef diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-llrint.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-llrint.mir new file mode 100644 index 000000000000..f77649b79395 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-llrint.mir @@ -0,0 +1,98 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -verify-machineinstrs -mtriple aarch64-unknown-unknown -run-pass=legalizer %s -o - | FileCheck %s +--- +name: testmsws +alignment: 4 +tracksRegLiveness: true +liveins: + - { reg: '$s0' } +body: | + bb.1: + liveins: $s0 + + ; CHECK-LABEL: name: testmsws + ; CHECK: liveins: $s0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $s0 + ; CHECK-NEXT: [[INTRINSIC_LLRINT:%[0-9]+]]:_(s64) = G_INTRINSIC_LLRINT [[COPY]](s32) + ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[INTRINSIC_LLRINT]](s64) + ; CHECK-NEXT: $w0 = COPY [[TRUNC]](s32) + ; CHECK-NEXT: RET_ReallyLR implicit $w0 + %0:_(s32) = COPY $s0 + %1:_(s64) = G_INTRINSIC_LLRINT %0(s32) + %2:_(s32) = G_TRUNC %1(s64) + $w0 = COPY %2(s32) + RET_ReallyLR implicit $w0 + +... +--- +name: testmsxs +alignment: 4 +tracksRegLiveness: true +liveins: + - { reg: '$s0' } +body: | + bb.1: + liveins: $s0 + + ; CHECK-LABEL: name: testmsxs + ; CHECK: liveins: $s0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $s0 + ; CHECK-NEXT: [[INTRINSIC_LLRINT:%[0-9]+]]:_(s64) = G_INTRINSIC_LLRINT [[COPY]](s32) + ; CHECK-NEXT: $x0 = COPY [[INTRINSIC_LLRINT]](s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %0:_(s32) = COPY $s0 + %1:_(s64) = G_INTRINSIC_LLRINT %0(s32) + $x0 = COPY %1(s64) + RET_ReallyLR implicit $x0 + +... +--- +name: testmswd +alignment: 4 +tracksRegLiveness: true +liveins: + - { reg: '$d0' } +body: | + bb.1: + liveins: $d0 + + ; CHECK-LABEL: name: testmswd + ; CHECK: liveins: $d0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $d0 + ; CHECK-NEXT: [[INTRINSIC_LLRINT:%[0-9]+]]:_(s64) = G_INTRINSIC_LLRINT [[COPY]](s64) + ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[INTRINSIC_LLRINT]](s64) + ; CHECK-NEXT: $w0 = COPY [[TRUNC]](s32) + ; CHECK-NEXT: RET_ReallyLR implicit $w0 + %0:_(s64) = COPY $d0 + %1:_(s64) = G_INTRINSIC_LLRINT %0(s64) + %2:_(s32) = G_TRUNC %1(s64) + $w0 = COPY %2(s32) + RET_ReallyLR implicit $w0 + +... +--- +name: testmsxd +alignment: 4 +tracksRegLiveness: true +liveins: + - { reg: '$d0' } +body: | + bb.1: + liveins: $d0 + + ; CHECK-LABEL: name: testmsxd + ; CHECK: liveins: $d0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $d0 + ; CHECK-NEXT: [[INTRINSIC_LLRINT:%[0-9]+]]:_(s64) = G_INTRINSIC_LLRINT [[COPY]](s64) + ; CHECK-NEXT: $x0 = COPY [[INTRINSIC_LLRINT]](s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %0:_(s64) = COPY $d0 + %1:_(s64) = G_INTRINSIC_LLRINT %0(s64) + $x0 = COPY %1(s64) + RET_ReallyLR implicit $x0 + +... diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir index 0793f3983c8e..098726b0a980 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir @@ -156,6 +156,10 @@ # DEBUG-NEXT: G_INTRINSIC_LRINT (opcode {{[0-9]+}}): 2 type indices, 0 imm indices # DEBUG-NEXT: .. the first uncovered type index: 2, OK # DEBUG-NEXT: .. the first uncovered imm index: 0, OK +# DEBUG-NEXT: G_INTRINSIC_LLRINT (opcode {{[0-9]+}}): 2 type indices, 0 imm indices +# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}} +# DEBUG-NEXT: .. the first uncovered type index: 2, OK +# DEBUG-NEXT: .. the first uncovered imm index: 0, OK # DEBUG-NEXT: G_INTRINSIC_ROUNDEVEN (opcode {{[0-9]+}}): 1 type index, 0 imm indices # DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}} # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected diff --git a/llvm/test/CodeGen/AArch64/llrint-conv-fp16.ll b/llvm/test/CodeGen/AArch64/llrint-conv-fp16.ll index 1adbbab76abf..7e28c863b07a 100644 --- a/llvm/test/CodeGen/AArch64/llrint-conv-fp16.ll +++ b/llvm/test/CodeGen/AArch64/llrint-conv-fp16.ll @@ -1,6 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2 ; RUN: llc < %s -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK-NOFP16 ; RUN: llc < %s -mtriple=aarch64 -mattr=+fullfp16 | FileCheck %s --check-prefixes=CHECK-FP16 +; RUN: llc < %s -mtriple=aarch64 -global-isel | FileCheck %s --check-prefixes=CHECK-NOFP16 +; RUN: llc < %s -mtriple=aarch64 -mattr=+fullfp16 -global-isel | FileCheck %s --check-prefixes=CHECK-FP16 define i16 @testmhhs(half %x) { ; CHECK-NOFP16-LABEL: testmhhs: diff --git a/llvm/test/CodeGen/AArch64/llrint-conv.ll b/llvm/test/CodeGen/AArch64/llrint-conv.ll index fa11b007eeb3..3a6396d120f7 100644 --- a/llvm/test/CodeGen/AArch64/llrint-conv.ll +++ b/llvm/test/CodeGen/AArch64/llrint-conv.ll @@ -1,59 +1,75 @@ -; RUN: llc < %s -mtriple=aarch64 -mattr=+neon | FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2 +; RUN: llc < %s -mtriple=aarch64 | FileCheck %s +; RUN: llc < %s -mtriple=aarch64 -global-isel | FileCheck %s -; CHECK-LABEL: testmsws: -; CHECK: frintx [[REG:s[0-9]]], s0 -; CHECK-NEXT: fcvtzs x0, [[REG]] -; CHECK: ret define i32 @testmsws(float %x) { +; CHECK-LABEL: testmsws: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: frintx s0, s0 +; CHECK-NEXT: fcvtzs x0, s0 +; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0 +; CHECK-NEXT: ret entry: %0 = tail call i64 @llvm.llrint.f32(float %x) %conv = trunc i64 %0 to i32 ret i32 %conv } -; CHECK-LABEL: testmsxs: -; CHECK: frintx [[REG:s[0-9]]], s0 -; CHECK-NEXT: fcvtzs x0, [[REG]] -; CHECK-NEXT: ret define i64 @testmsxs(float %x) { +; CHECK-LABEL: testmsxs: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: frintx s0, s0 +; CHECK-NEXT: fcvtzs x0, s0 +; CHECK-NEXT: ret entry: %0 = tail call i64 @llvm.llrint.f32(float %x) ret i64 %0 } -; CHECK-LABEL: testmswd: -; CHECK: frintx [[REG:d[0-9]]], d0 -; CHECK-NEXT: fcvtzs x0, [[REG]] -; CHECK: ret define i32 @testmswd(double %x) { +; CHECK-LABEL: testmswd: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: frintx d0, d0 +; CHECK-NEXT: fcvtzs x0, d0 +; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0 +; CHECK-NEXT: ret entry: %0 = tail call i64 @llvm.llrint.f64(double %x) %conv = trunc i64 %0 to i32 ret i32 %conv } -; CHECK-LABEL: testmsxd: -; CHECK: frintx [[REG:d[0-9]]], d0 -; CHECK-NEXT: fcvtzs x0, [[REG]] -; CHECK-nEXT: ret define i64 @testmsxd(double %x) { +; CHECK-LABEL: testmsxd: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: frintx d0, d0 +; CHECK-NEXT: fcvtzs x0, d0 +; CHECK-NEXT: ret entry: %0 = tail call i64 @llvm.llrint.f64(double %x) ret i64 %0 } -; CHECK-LABEL: testmswl: -; CHECK: bl llrintl define i32 @testmswl(fp128 %x) { +; CHECK-LABEL: testmswl: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: .cfi_offset w30, -16 +; CHECK-NEXT: bl llrintl +; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0 +; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: ret entry: %0 = tail call i64 @llvm.llrint.f128(fp128 %x) %conv = trunc i64 %0 to i32 ret i32 %conv } -; CHECK-LABEL: testmsll: -; CHECK: b llrintl define i64 @testmsll(fp128 %x) { +; CHECK-LABEL: testmsll: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: b llrintl entry: %0 = tail call i64 @llvm.llrint.f128(fp128 %x) ret i64 %0 diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-replacerreg.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-replacerreg.td index 40a831d7e9e8..ebb95ccb2104 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-replacerreg.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-replacerreg.td @@ -28,11 +28,11 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK: const uint8_t *GenMyCombiner::getMatchTable() const { // CHECK-NEXT: constexpr static uint8_t MatchTable0[] = { -// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(69), GIMT_Encode2(186), /*)*//*default:*//*Label 2*/ GIMT_Encode4(562), -// CHECK-NEXT: /*TargetOpcode::G_UNMERGE_VALUES*//*Label 0*/ GIMT_Encode4(478), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_FNEG*//*Label 1*/ GIMT_Encode4(530), -// CHECK-NEXT: // Label 0: @478 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 3*/ GIMT_Encode4(529), // Rule ID 1 // +// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2({{[0-9]+}}), GIMT_Encode2({{[0-9]+}}), /*)*//*default:*//*Label 2*/ GIMT_Encode4([[L562:[0-9]+]]), +// CHECK-NEXT: /*TargetOpcode::G_UNMERGE_VALUES*//*Label 0*/ GIMT_Encode4([[L478:[0-9]+]]), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_FNEG*//*Label 1*/ GIMT_Encode4([[L530:[0-9]+]]), +// CHECK-NEXT: // Label 0: @[[L478]] +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 3*/ GIMT_Encode4([[L529:[0-9]+]]), // Rule ID 1 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled), // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // CHECK-NEXT: // MIs[0] a @@ -57,10 +57,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_ReplaceRegWithTempReg, /*OldInsnID*/0, /*OldOpIdx*/1, /*TempRegID*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 3: @529 +// CHECK-NEXT: // Label 3: @[[L529]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 1: @530 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4(561), // Rule ID 0 // +// CHECK-NEXT: // Label 1: @[[L530]] +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4([[L561:[0-9]+]]), // Rule ID 0 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled), // CHECK-NEXT: // MIs[0] dst // CHECK-NEXT: // No operand predicates @@ -75,10 +75,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_ReplaceReg, /*OldInsnID*/0, /*OldOpIdx*/0, /*NewInsnId*/1, /*NewOpIdx*/1, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 4: @561 +// CHECK-NEXT: // Label 4: @[[L561]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 2: @562 +// CHECK-NEXT: // Label 2: @[[L562]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: }; // Size: 563 bytes +// CHECK-NEXT: }; // Size: {{[0-9]+}} bytes // CHECK-NEXT: return MatchTable0; // CHECK-NEXT: } diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td index 751b1318ecc0..6004a17d351b 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td @@ -34,12 +34,12 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK: const uint8_t *GenMyCombiner::getMatchTable() const { // CHECK-NEXT: constexpr static uint8_t MatchTable0[] = { -// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(19), GIMT_Encode2(132), /*)*//*default:*//*Label 3*/ GIMT_Encode4(579), -// CHECK-NEXT: /*TargetOpcode::COPY*//*Label 0*/ GIMT_Encode4(462), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_CONSTANT*//*Label 1*/ GIMT_Encode4(493), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 2*/ GIMT_Encode4(539), -// CHECK-NEXT: // Label 0: @462 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4(492), // Rule ID 0 // +// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(19), GIMT_Encode2({{[0-9]+}}), /*)*//*default:*//*Label 3*/ GIMT_Encode4([[L579:[0-9]+]]), +// CHECK-NEXT: /*TargetOpcode::COPY*//*Label 0*/ GIMT_Encode4([[L462:[0-9]+]]), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_CONSTANT*//*Label 1*/ GIMT_Encode4([[L493:[0-9]+]]), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 2*/ GIMT_Encode4({{[0-9]+}}), +// CHECK-NEXT: // Label 0: @[[L462]] +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4([[L492:[0-9]+]]), // Rule ID 0 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled), // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32, // CHECK-NEXT: // MIs[0] a @@ -51,10 +51,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_AddImm8, /*InsnID*/0, /*Imm*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 4: @492 +// CHECK-NEXT: // Label 4: @[[L492]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 1: @493 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 5*/ GIMT_Encode4(538), // Rule ID 2 // +// CHECK-NEXT: // Label 1: @[[L493]] +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 5*/ GIMT_Encode4([[L538:[0-9]+]]), // Rule ID 2 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule2Enabled), // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32, // CHECK-NEXT: // MIs[0] a @@ -66,10 +66,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_AddCImm, /*InsnID*/0, /*Type*/GILLT_s32, /*Imm*/GIMT_Encode8(42), // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 5: @538 +// CHECK-NEXT: // Label 5: @[[L538]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 2: @539 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 6*/ GIMT_Encode4(578), // Rule ID 1 // +// CHECK-NEXT: // Label 2: @{{[0-9]+}} +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 6*/ GIMT_Encode4([[L578:[0-9]+]]), // Rule ID 1 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled), // CHECK-NEXT: // MIs[0] a // CHECK-NEXT: // No operand predicates @@ -83,10 +83,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 6: @578 +// CHECK-NEXT: // Label 6: @[[L578]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 3: @579 +// CHECK-NEXT: // Label 3: @[[L579]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: }; // Size: 580 bytes +// CHECK-NEXT: }; // Size: {{[0-9]+}} bytes // CHECK-NEXT: return MatchTable0; // CHECK-NEXT: } diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-intrinsics.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-intrinsics.td index e8e6d3e74f40..b2dd8b6684b1 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-intrinsics.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-intrinsics.td @@ -29,11 +29,11 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK: const uint8_t *GenMyCombiner::getMatchTable() const { // CHECK-NEXT: constexpr static uint8_t MatchTable0[] = { -// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(119), GIMT_Encode2(121), /*)*//*default:*//*Label 2*/ GIMT_Encode4(132), +// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2({{[0-9]+}}), GIMT_Encode2({{[0-9]+}}), /*)*//*default:*//*Label 2*/ GIMT_Encode4([[L132:[0-9]+]]), // CHECK-NEXT: /*TargetOpcode::G_INTRINSIC*//*Label 0*/ GIMT_Encode4(18), -// CHECK-NEXT: /*TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS*//*Label 1*/ GIMT_Encode4(73), +// CHECK-NEXT: /*TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS*//*Label 1*/ GIMT_Encode4([[L73:[0-9]+]]), // CHECK-NEXT: // Label 0: @18 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 3*/ GIMT_Encode4(72), // Rule ID 0 // +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 3*/ GIMT_Encode4([[L72:[0-9]+]]), // Rule ID 0 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled), // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // CHECK-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode2(Intrinsic::1in_1out), @@ -52,10 +52,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/1, /*TempRegID*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 3: @72 +// CHECK-NEXT: // Label 3: @[[L72]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 1: @73 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4(131), // Rule ID 1 // +// CHECK-NEXT: // Label 1: @[[L73]] +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4([[L131:[0-9]+]]), // Rule ID 1 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled), // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // CHECK-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode2(Intrinsic::sideeffects_1in_1out), @@ -76,9 +76,9 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_MergeMemOperands, /*InsnID*/1, /*NumInsns*/1, /*MergeInsnID's*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 4: @131 +// CHECK-NEXT: // Label 4: @[[L131]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 2: @132 +// CHECK-NEXT: // Label 2: @[[L132]] // CHECK-NEXT: GIM_Reject, // CHECK-NEXT: }; // Size: 133 bytes // CHECK-NEXT: return MatchTable0; diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td index 26a0ec6235e3..016ab05ca01e 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td @@ -28,12 +28,12 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK: const uint8_t *GenMyCombiner::getMatchTable() const { // CHECK-NEXT: constexpr static uint8_t MatchTable0[] = { -// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(124), GIMT_Encode2(187), /*)*//*default:*//*Label 3*/ GIMT_Encode4(380), -// CHECK-NEXT: /*TargetOpcode::G_TRUNC*//*Label 0*/ GIMT_Encode4(262), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 1*/ GIMT_Encode4(298), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_FPEXT*//*Label 2*/ GIMT_Encode4(344), -// CHECK-NEXT: // Label 0: @262 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4(297), // Rule ID 1 // +// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2({{[0-9]+}}), GIMT_Encode2({{[0-9]+}}), /*)*//*default:*//*Label 3*/ GIMT_Encode4([[L380:[0-9]+]]), +// CHECK-NEXT: /*TargetOpcode::G_TRUNC*//*Label 0*/ GIMT_Encode4([[L262:[0-9]+]]), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 1*/ GIMT_Encode4([[L298:[0-9]+]]), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_FPEXT*//*Label 2*/ GIMT_Encode4([[L344:[0-9]+]]), +// CHECK-NEXT: // Label 0: @[[L262]] +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4([[L297:[0-9]+]]), // Rule ID 1 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled), // CHECK-NEXT: // MIs[0] root // CHECK-NEXT: // No operand predicates @@ -47,10 +47,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 4: @297 +// CHECK-NEXT: // Label 4: @[[L297]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 1: @298 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 5*/ GIMT_Encode4(343), // Rule ID 0 // +// CHECK-NEXT: // Label 1: @[[L298]] +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 5*/ GIMT_Encode4([[L343:[0-9]+]]), // Rule ID 0 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled), // CHECK-NEXT: // MIs[0] root // CHECK-NEXT: // No operand predicates @@ -68,10 +68,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 5: @343 +// CHECK-NEXT: // Label 5: @[[L343]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 2: @344 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 6*/ GIMT_Encode4(379), // Rule ID 2 // +// CHECK-NEXT: // Label 2: @[[L344]] +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 6*/ GIMT_Encode4([[L379:[0-9]+]]), // Rule ID 2 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled), // CHECK-NEXT: // MIs[0] root // CHECK-NEXT: // No operand predicates @@ -85,10 +85,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 6: @379 +// CHECK-NEXT: // Label 6: @[[L379]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 3: @380 +// CHECK-NEXT: // Label 3: @[[L380]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: }; // Size: 381 bytes +// CHECK-NEXT: }; // Size: {{[0-9]+}} bytes // CHECK-NEXT: return MatchTable0; // CHECK-NEXT: } diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td index 0189d3d056fc..02085c1fd266 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td @@ -135,15 +135,15 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // Verify match table. // CHECK: const uint8_t *GenMyCombiner::getMatchTable() const { // CHECK-NEXT: constexpr static uint8_t MatchTable0[] = { -// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(19), GIMT_Encode2(132), /*)*//*default:*//*Label 6*/ GIMT_Encode4(677), -// CHECK-NEXT: /*TargetOpcode::COPY*//*Label 0*/ GIMT_Encode4(462), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_AND*//*Label 1*/ GIMT_Encode4(504), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_STORE*//*Label 2*/ GIMT_Encode4(557), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_TRUNC*//*Label 3*/ GIMT_Encode4(599), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_SEXT*//*Label 4*/ GIMT_Encode4(624), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 5*/ GIMT_Encode4(637), -// CHECK-NEXT: // Label 0: @462 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 7*/ GIMT_Encode4(491), // Rule ID 4 // +// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(19), GIMT_Encode2({{[0-9]+}}), /*)*//*default:*//*Label 6*/ GIMT_Encode4([[L677:[0-9]+]]), +// CHECK-NEXT: /*TargetOpcode::COPY*//*Label 0*/ GIMT_Encode4([[L462:[0-9]+]]), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_AND*//*Label 1*/ GIMT_Encode4([[L504:[0-9]+]]), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_STORE*//*Label 2*/ GIMT_Encode4([[L557:[0-9]+]]), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_TRUNC*//*Label 3*/ GIMT_Encode4([[L599:[0-9]+]]), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_SEXT*//*Label 4*/ GIMT_Encode4([[L624:[0-9]+]]), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 5*/ GIMT_Encode4([[L637:[0-9]+]]), +// CHECK-NEXT: // Label 0: @[[L462]] +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 7*/ GIMT_Encode4([[L491:[0-9]+]]), // Rule ID 4 // // CHECK-NEXT: GIM_CheckFeatures, GIMT_Encode2(GIFBS_HasAnswerToEverything), // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule3Enabled), // CHECK-NEXT: // MIs[0] a @@ -158,8 +158,8 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: // Combiner Rule #3: InstTest1 // CHECK-NEXT: GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0), // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 7: @491 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 8*/ GIMT_Encode4(503), // Rule ID 3 // +// CHECK-NEXT: // Label 7: @[[L491]] +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 8*/ GIMT_Encode4([[L503:[0-9]+]]), // Rule ID 3 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule2Enabled), // CHECK-NEXT: // MIs[0] a // CHECK-NEXT: // No operand predicates @@ -168,10 +168,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: // Combiner Rule #2: InstTest0 // CHECK-NEXT: GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner1), // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 8: @503 +// CHECK-NEXT: // Label 8: @[[L503]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 1: @504 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 9*/ GIMT_Encode4(556), // Rule ID 6 // +// CHECK-NEXT: // Label 1: @[[L504]] +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 9*/ GIMT_Encode4([[L556:[0-9]+]]), // Rule ID 6 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule5Enabled), // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32, // CHECK-NEXT: // MIs[0] dst @@ -189,10 +189,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // z // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 9: @556 +// CHECK-NEXT: // Label 9: @[[L556]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 2: @557 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 10*/ GIMT_Encode4(598), // Rule ID 5 // +// CHECK-NEXT: // Label 2: @[[L557]] +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 10*/ GIMT_Encode4([[L598:[0-9]+]]), // Rule ID 5 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule4Enabled), // CHECK-NEXT: // MIs[0] tmp // CHECK-NEXT: GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/0, // MIs[1] @@ -210,32 +210,32 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner2), // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 10: @598 +// CHECK-NEXT: // Label 10: @[[L598]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 3: @599 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 11*/ GIMT_Encode4(611), // Rule ID 0 // +// CHECK-NEXT: // Label 3: @[[L599]] +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 11*/ GIMT_Encode4([[L611:[0-9]+]]), // Rule ID 0 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled), // CHECK-NEXT: // Combiner Rule #0: WipOpcodeTest0; wip_match_opcode 'G_TRUNC' // CHECK-NEXT: GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0), // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 11: @611 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 12*/ GIMT_Encode4(623), // Rule ID 1 // +// CHECK-NEXT: // Label 11: @[[L611]] +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 12*/ GIMT_Encode4([[L623:[0-9]+]]), // Rule ID 1 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled), // CHECK-NEXT: // Combiner Rule #1: WipOpcodeTest1; wip_match_opcode 'G_TRUNC' // CHECK-NEXT: GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0), // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 12: @623 +// CHECK-NEXT: // Label 12: @[[L623]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 4: @624 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 13*/ GIMT_Encode4(636), // Rule ID 2 // +// CHECK-NEXT: // Label 4: @[[L624]] +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 13*/ GIMT_Encode4([[L636:[0-9]+]]), // Rule ID 2 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled), // CHECK-NEXT: // Combiner Rule #1: WipOpcodeTest1; wip_match_opcode 'G_SEXT' // CHECK-NEXT: GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0), // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 13: @636 +// CHECK-NEXT: // Label 13: @[[L636]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 5: @637 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 14*/ GIMT_Encode4(676), // Rule ID 7 // +// CHECK-NEXT: // Label 5: @[[L637]] +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 14*/ GIMT_Encode4([[L676:[0-9]+]]), // Rule ID 7 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule6Enabled), // CHECK-NEXT: // MIs[0] dst // CHECK-NEXT: // No operand predicates @@ -250,10 +250,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 14: @676 +// CHECK-NEXT: // Label 14: @[[L676]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 6: @677 +// CHECK-NEXT: // Label 6: @[[L677]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: }; // Size: 678 bytes +// CHECK-NEXT: }; // Size: {{[0-9]+}} bytes // CHECK-NEXT: return MatchTable0; // CHECK-NEXT: } diff --git a/llvm/test/TableGen/GlobalISelEmitter.td b/llvm/test/TableGen/GlobalISelEmitter.td index f79b792b37a3..82ecc4495e80 100644 --- a/llvm/test/TableGen/GlobalISelEmitter.td +++ b/llvm/test/TableGen/GlobalISelEmitter.td @@ -518,7 +518,7 @@ def : Pat<(frag GPR32:$src1, complex:$src2, complex:$src3), // R00O-NEXT: GIM_Reject, // R00O: // Label [[DEFAULT_NUM]]: @[[DEFAULT]] // R00O-NEXT: GIM_Reject, -// R00O-NEXT: }; // Size: 2023 bytes +// R00O-NEXT: }; // Size: 2027 bytes def INSNBOB : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2, GPR32:$src3, GPR32:$src4), [(set GPR32:$dst, -- GitLab From 2c22a0c16d1cb844eac142156ba67098627a336c Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Fri, 12 Apr 2024 12:35:40 -0700 Subject: [PATCH 084/862] [InstCombine] Add test case for turning sub into xor using dominating condition. NFC I plan to disable using dominating conditions for turning sub into xor, but first we need that demonstrates it currently happens. --- llvm/test/Transforms/InstCombine/sub-xor.ll | 23 +++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/llvm/test/Transforms/InstCombine/sub-xor.ll b/llvm/test/Transforms/InstCombine/sub-xor.ll index 71da73d51ae3..b4e87d0405fc 100644 --- a/llvm/test/Transforms/InstCombine/sub-xor.ll +++ b/llvm/test/Transforms/InstCombine/sub-xor.ll @@ -157,3 +157,26 @@ define <2 x i8> @xor_add_splat_undef(<2 x i8> %x) { %add = add <2 x i8> %xor, ret <2 x i8> %add } + +define i32 @xor_dominating_cond(i32 %x) { +; CHECK-LABEL: @xor_dominating_cond( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[COND:%.*]] = icmp ult i32 [[X:%.*]], 256 +; CHECK-NEXT: br i1 [[COND]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] +; CHECK: if.then: +; CHECK-NEXT: [[A:%.*]] = xor i32 [[X]], 255 +; CHECK-NEXT: ret i32 [[A]] +; CHECK: if.end: +; CHECK-NEXT: ret i32 [[X]] +; +entry: + %cond = icmp ult i32 %x, 256 + br i1 %cond, label %if.then, label %if.end + +if.then: + %a = sub i32 255, %x + ret i32 %a + +if.end: + ret i32 %x +} -- GitLab From 421a8c5892b7e59f27b2c21452f81fa789a758fd Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Tue, 16 Apr 2024 21:25:28 -0700 Subject: [PATCH 085/862] [InstCombine] Add phase ordering test for #88239. NFC --- .../Transforms/PhaseOrdering/X86/pr88239.ll | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 llvm/test/Transforms/PhaseOrdering/X86/pr88239.ll diff --git a/llvm/test/Transforms/PhaseOrdering/X86/pr88239.ll b/llvm/test/Transforms/PhaseOrdering/X86/pr88239.ll new file mode 100644 index 000000000000..3afa1904fb24 --- /dev/null +++ b/llvm/test/Transforms/PhaseOrdering/X86/pr88239.ll @@ -0,0 +1,55 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt < %s -passes="default" -mcpu=skx -S | FileCheck %s + +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define void @foo(ptr noalias noundef %0, ptr noalias noundef %1) optsize { +; CHECK-LABEL: define void @foo( +; CHECK-SAME: ptr noalias nocapture noundef readonly [[TMP0:%.*]], ptr noalias nocapture noundef writeonly [[TMP1:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: br label [[TMP4:%.*]] +; CHECK: vector.body: +; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[TMP2:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[TMP4]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi <8 x i64> [ , [[TMP2]] ], [ [[VEC_IND_NEXT:%.*]], [[TMP4]] ] +; CHECK-NEXT: [[TMP6:%.*]] = and <8 x i64> [[VEC_IND]], +; CHECK-NEXT: [[TMP3:%.*]] = xor <8 x i64> [[TMP6]], +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], <8 x i64> [[TMP3]] +; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = tail call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> [[TMP7]], i32 4, <8 x i1> , <8 x i32> poison) +; CHECK-NEXT: [[TMP5:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_GATHER]], +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 [[INDVARS_IV]] +; CHECK-NEXT: store <8 x i32> [[TMP5]], ptr [[TMP10]], align 4 +; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw i64 [[INDVARS_IV]], 8 +; CHECK-NEXT: [[VEC_IND_NEXT]] = add <8 x i64> [[VEC_IND]], +; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 256 +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[MIDDLE_BLOCK:%.*]], label [[TMP4]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK: middle.block: +; CHECK-NEXT: ret void +; + br label %3 + +3: ; preds = %7, %2 + %4 = phi i32 [ 0, %2 ], [ %15, %7 ] + %5 = icmp slt i32 %4, 256 + br i1 %5, label %7, label %6 + +6: ; preds = %3 + ret void + +7: ; preds = %3 + %8 = sub nsw i32 255, %4 + %9 = zext nneg i32 %8 to i64 + %10 = getelementptr inbounds i32, ptr %0, i64 %9 + %11 = load i32, ptr %10, align 4 + %12 = add nsw i32 %11, 5 + %13 = sext i32 %4 to i64 + %14 = getelementptr inbounds i32, ptr %1, i64 %13 + store i32 %12, ptr %14, align 4 + %15 = add nsw i32 %4, 1 + br label %3 +} +;. +; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]} +; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1} +; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"} +;. -- GitLab From ed741ffe893698cd14c6785ac2ee7031d9d344a6 Mon Sep 17 00:00:00 2001 From: Nathan Lanza Date: Wed, 17 Apr 2024 14:06:01 -0400 Subject: [PATCH 086/862] [github] Add ClangIR to new-prs-labeler.yml (#86088) --- .github/new-prs-labeler.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/new-prs-labeler.yml b/.github/new-prs-labeler.yml index a0428336d300..1502d64a7d3e 100644 --- a/.github/new-prs-labeler.yml +++ b/.github/new-prs-labeler.yml @@ -1,3 +1,9 @@ +ClangIR: + - clang/include/clang/CIR/**/* + - clang/lib/CIR/**/* + - clang/tools/cir-*/**/* + - clang/test/CIR/**/* + clang:dataflow: - clang/include/clang/Analysis/FlowSensitive/**/* - clang/lib/Analysis/FlowSensitive/**/* -- GitLab From c02ed29ec151d1d555c3735efef2ab215126ddbf Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Wed, 17 Apr 2024 19:01:44 +0100 Subject: [PATCH 087/862] [CostModel][X86] Recognise vector rotation by uniform constant patterns Adds suitable costs for AVX512 targets (we still rely on default expansion for AVX2 and earlier) --- .../lib/Target/X86/X86TargetTransformInfo.cpp | 41 +++++++++++++--- .../Analysis/CostModel/X86/fshl-codesize.ll | 24 +++++----- .../Analysis/CostModel/X86/fshl-latency.ll | 24 +++++----- .../CostModel/X86/fshl-sizelatency.ll | 24 +++++----- llvm/test/Analysis/CostModel/X86/fshl.ll | 22 ++++----- .../Analysis/CostModel/X86/fshr-codesize.ll | 48 +++++++++---------- .../Analysis/CostModel/X86/fshr-latency.ll | 24 +++++----- .../CostModel/X86/fshr-sizelatency.ll | 48 +++++++++---------- llvm/test/Analysis/CostModel/X86/fshr.ll | 22 ++++----- .../SLPVectorizer/X86/arith-fshl-rot.ll | 40 ++++++++++------ .../SLPVectorizer/X86/arith-fshr-rot.ll | 40 ++++++++++------ 11 files changed, 204 insertions(+), 153 deletions(-) diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp index 38064f979269..d111c4d4ecc1 100644 --- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp @@ -3402,6 +3402,9 @@ X86TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, { ISD::ROTR, MVT::v32i16, { 1, 1, 1, 1 } }, { ISD::ROTR, MVT::v16i16, { 1, 1, 1, 1 } }, { ISD::ROTR, MVT::v8i16, { 1, 1, 1, 1 } }, + { X86ISD::VROTLI, MVT::v32i16, { 1, 1, 1, 1 } }, + { X86ISD::VROTLI, MVT::v16i16, { 1, 1, 1, 1 } }, + { X86ISD::VROTLI, MVT::v8i16, { 1, 1, 1, 1 } }, }; static const CostKindTblEntry AVX512BITALGCostTbl[] = { { ISD::CTPOP, MVT::v32i16, { 1, 1, 1, 1 } }, @@ -3498,6 +3501,12 @@ X86TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, { ISD::ROTR, MVT::v64i8, { 5, 6, 12, 14 } }, { ISD::ROTR, MVT::v32i8, { 5, 14, 6, 9 } }, { ISD::ROTR, MVT::v16i8, { 5, 14, 6, 9 } }, + { X86ISD::VROTLI, MVT::v32i16, { 2, 5, 3, 3 } }, + { X86ISD::VROTLI, MVT::v16i16, { 1, 5, 3, 3 } }, + { X86ISD::VROTLI, MVT::v8i16, { 1, 5, 3, 3 } }, + { X86ISD::VROTLI, MVT::v64i8, { 2, 9, 3, 4 } }, + { X86ISD::VROTLI, MVT::v32i8, { 1, 9, 3, 4 } }, + { X86ISD::VROTLI, MVT::v16i8, { 1, 8, 3, 4 } }, { ISD::SADDSAT, MVT::v32i16, { 1 } }, { ISD::SADDSAT, MVT::v64i8, { 1 } }, { ISD::SMAX, MVT::v32i16, { 1, 1, 1, 1 } }, @@ -3556,6 +3565,12 @@ X86TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, { ISD::ROTR, MVT::v16i32, { 1, 1, 1, 1 } }, { ISD::ROTR, MVT::v8i32, { 1, 1, 1, 1 } }, { ISD::ROTR, MVT::v4i32, { 1, 1, 1, 1 } }, + { X86ISD::VROTLI, MVT::v8i64, { 1, 1, 1, 1 } }, + { X86ISD::VROTLI, MVT::v4i64, { 1, 1, 1, 1 } }, + { X86ISD::VROTLI, MVT::v2i64, { 1, 1, 1, 1 } }, + { X86ISD::VROTLI, MVT::v16i32, { 1, 1, 1, 1 } }, + { X86ISD::VROTLI, MVT::v8i32, { 1, 1, 1, 1 } }, + { X86ISD::VROTLI, MVT::v4i32, { 1, 1, 1, 1 } }, { ISD::SMAX, MVT::v8i64, { 1, 3, 1, 1 } }, { ISD::SMAX, MVT::v16i32, { 1, 1, 1, 1 } }, { ISD::SMAX, MVT::v32i16, { 3, 7, 5, 5 } }, @@ -3642,7 +3657,15 @@ X86TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, { ISD::ROTR, MVT::v2i64, { 1, 3, 3, 3 } }, { ISD::ROTR, MVT::v4i32, { 1, 3, 3, 3 } }, { ISD::ROTR, MVT::v8i16, { 1, 3, 3, 3 } }, - { ISD::ROTR, MVT::v16i8, { 1, 3, 3, 3 } } + { ISD::ROTR, MVT::v16i8, { 1, 3, 3, 3 } }, + { X86ISD::VROTLI, MVT::v4i64, { 4, 7, 5, 6 } }, + { X86ISD::VROTLI, MVT::v8i32, { 4, 7, 5, 6 } }, + { X86ISD::VROTLI, MVT::v16i16, { 4, 7, 5, 6 } }, + { X86ISD::VROTLI, MVT::v32i8, { 4, 7, 5, 6 } }, + { X86ISD::VROTLI, MVT::v2i64, { 1, 3, 1, 1 } }, + { X86ISD::VROTLI, MVT::v4i32, { 1, 3, 1, 1 } }, + { X86ISD::VROTLI, MVT::v8i16, { 1, 3, 1, 1 } }, + { X86ISD::VROTLI, MVT::v16i8, { 1, 3, 1, 1 } }, }; static const CostKindTblEntry AVX2CostTbl[] = { { ISD::ABS, MVT::v2i64, { 2, 4, 3, 5 } }, // VBLENDVPD(X,VPSUBQ(0,X),X) @@ -4096,9 +4119,11 @@ X86TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, const SmallVectorImpl &Args = ICA.getArgs(); if (Args[0] == Args[1]) { ISD = ISD::ROTL; - // Handle scalar constant rotation amounts. - // TODO: Handle vector + funnel-shift cases. - if (isa_and_nonnull(Args[2])) + // Handle uniform constant rotation amounts. + // TODO: Handle funnel-shift cases. + const APInt *Amt; + if (Args[2] && + PatternMatch::match(Args[2], PatternMatch::m_APIntAllowUndef(Amt))) ISD = X86ISD::VROTLI; } } @@ -4109,10 +4134,12 @@ X86TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, if (!ICA.isTypeBasedOnly()) { const SmallVectorImpl &Args = ICA.getArgs(); if (Args[0] == Args[1]) { - // Handle scalar constant rotation amount. - // TODO: Handle vector + funnel-shift cases. ISD = ISD::ROTR; - if (isa_and_nonnull(Args[2])) + // Handle uniform constant rotation amount. + // TODO: Handle funnel-shift cases. + const APInt *Amt; + if (Args[2] && + PatternMatch::match(Args[2], PatternMatch::m_APIntAllowUndef(Amt))) ISD = X86ISD::VROTLI; } } diff --git a/llvm/test/Analysis/CostModel/X86/fshl-codesize.ll b/llvm/test/Analysis/CostModel/X86/fshl-codesize.ll index c46e32ffb4ad..a7585a4d9f39 100644 --- a/llvm/test/Analysis/CostModel/X86/fshl-codesize.ll +++ b/llvm/test/Analysis/CostModel/X86/fshl-codesize.ll @@ -2744,9 +2744,9 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; ; AVX512BW-LABEL: 'splatconstant_rotate_i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 3) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512DQ-LABEL: 'splatconstant_rotate_i16' @@ -2829,9 +2829,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512BW-LABEL: 'splatconstant_rotate_i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512DQ-LABEL: 'splatconstant_rotate_i8' @@ -2843,9 +2843,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512VBMI2-LABEL: 'splatconstant_rotate_i8' ; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; SLM-LABEL: 'splatconstant_rotate_i8' @@ -2871,9 +2871,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512GFNI-LABEL: 'splatconstant_rotate_i8' ; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) diff --git a/llvm/test/Analysis/CostModel/X86/fshl-latency.ll b/llvm/test/Analysis/CostModel/X86/fshl-latency.ll index fa32497c63ec..7105f713fdc3 100644 --- a/llvm/test/Analysis/CostModel/X86/fshl-latency.ll +++ b/llvm/test/Analysis/CostModel/X86/fshl-latency.ll @@ -2696,9 +2696,9 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; ; AVX512BW-LABEL: 'splatconstant_rotate_i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 3) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512DQ-LABEL: 'splatconstant_rotate_i16' @@ -2781,9 +2781,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512BW-LABEL: 'splatconstant_rotate_i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512DQ-LABEL: 'splatconstant_rotate_i8' @@ -2795,9 +2795,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512VBMI2-LABEL: 'splatconstant_rotate_i8' ; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; SLM-LABEL: 'splatconstant_rotate_i8' @@ -2823,9 +2823,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512GFNI-LABEL: 'splatconstant_rotate_i8' ; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) diff --git a/llvm/test/Analysis/CostModel/X86/fshl-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/fshl-sizelatency.ll index 832a574a9b33..5d7361e29317 100644 --- a/llvm/test/Analysis/CostModel/X86/fshl-sizelatency.ll +++ b/llvm/test/Analysis/CostModel/X86/fshl-sizelatency.ll @@ -2984,9 +2984,9 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; ; AVX512BW-LABEL: 'splatconstant_rotate_i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 3) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512DQ-LABEL: 'splatconstant_rotate_i16' @@ -3069,9 +3069,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512BW-LABEL: 'splatconstant_rotate_i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512DQ-LABEL: 'splatconstant_rotate_i8' @@ -3083,9 +3083,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512VBMI2-LABEL: 'splatconstant_rotate_i8' ; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; SLM-LABEL: 'splatconstant_rotate_i8' @@ -3111,9 +3111,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512GFNI-LABEL: 'splatconstant_rotate_i8' ; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) diff --git a/llvm/test/Analysis/CostModel/X86/fshl.ll b/llvm/test/Analysis/CostModel/X86/fshl.ll index 311d8d5ed7d2..1cbdab09acd9 100644 --- a/llvm/test/Analysis/CostModel/X86/fshl.ll +++ b/llvm/test/Analysis/CostModel/X86/fshl.ll @@ -2682,8 +2682,8 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; ; AVX512BW-LABEL: 'splatconstant_rotate_i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 3) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; @@ -2767,9 +2767,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512BW-LABEL: 'splatconstant_rotate_i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; ; AVX512DQ-LABEL: 'splatconstant_rotate_i8' @@ -2781,9 +2781,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512VBMI2-LABEL: 'splatconstant_rotate_i8' ; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; ; SLM-LABEL: 'splatconstant_rotate_i8' @@ -2809,9 +2809,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512GFNI-LABEL: 'splatconstant_rotate_i8' ; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) diff --git a/llvm/test/Analysis/CostModel/X86/fshr-codesize.ll b/llvm/test/Analysis/CostModel/X86/fshr-codesize.ll index f9d30e4ced3e..ecc861dd7f8e 100644 --- a/llvm/test/Analysis/CostModel/X86/fshr-codesize.ll +++ b/llvm/test/Analysis/CostModel/X86/fshr-codesize.ll @@ -2644,9 +2644,9 @@ define void @splatconstant_rotate_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256 ; ; XOP-LABEL: 'splatconstant_rotate_i64' ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %a64, i64 7) -; XOP-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> ) -; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> ) -; XOP-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %a64, i64 7) @@ -2701,9 +2701,9 @@ define void @splatconstant_rotate_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256 ; ; XOP-LABEL: 'splatconstant_rotate_i32' ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %a32, i32 5) -; XOP-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> ) -; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> ) -; XOP-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %a32, i32 5) @@ -2744,9 +2744,9 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; ; AVX512BW-LABEL: 'splatconstant_rotate_i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 3) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512DQ-LABEL: 'splatconstant_rotate_i16' @@ -2779,9 +2779,9 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; ; XOP-LABEL: 'splatconstant_rotate_i16' ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 3) -; XOP-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) -; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) -; XOP-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512GFNI-LABEL: 'splatconstant_rotate_i16' @@ -2829,9 +2829,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512BW-LABEL: 'splatconstant_rotate_i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512DQ-LABEL: 'splatconstant_rotate_i8' @@ -2843,9 +2843,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512VBMI2-LABEL: 'splatconstant_rotate_i8' ; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; SLM-LABEL: 'splatconstant_rotate_i8' @@ -2864,16 +2864,16 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; XOP-LABEL: 'splatconstant_rotate_i8' ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) -; XOP-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; XOP-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512GFNI-LABEL: 'splatconstant_rotate_i8' ; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) diff --git a/llvm/test/Analysis/CostModel/X86/fshr-latency.ll b/llvm/test/Analysis/CostModel/X86/fshr-latency.ll index ed2227591847..0142ad77849c 100644 --- a/llvm/test/Analysis/CostModel/X86/fshr-latency.ll +++ b/llvm/test/Analysis/CostModel/X86/fshr-latency.ll @@ -2696,9 +2696,9 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; ; AVX512BW-LABEL: 'splatconstant_rotate_i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 3) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512DQ-LABEL: 'splatconstant_rotate_i16' @@ -2781,9 +2781,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512BW-LABEL: 'splatconstant_rotate_i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512DQ-LABEL: 'splatconstant_rotate_i8' @@ -2795,9 +2795,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512VBMI2-LABEL: 'splatconstant_rotate_i8' ; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; SLM-LABEL: 'splatconstant_rotate_i8' @@ -2823,9 +2823,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512GFNI-LABEL: 'splatconstant_rotate_i8' ; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) diff --git a/llvm/test/Analysis/CostModel/X86/fshr-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/fshr-sizelatency.ll index 8931781f70bd..6dafb20a0aee 100644 --- a/llvm/test/Analysis/CostModel/X86/fshr-sizelatency.ll +++ b/llvm/test/Analysis/CostModel/X86/fshr-sizelatency.ll @@ -2849,9 +2849,9 @@ define void @splatconstant_rotate_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256 ; ; XOP-LABEL: 'splatconstant_rotate_i64' ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %a64, i64 7) -; XOP-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> ) -; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> ) -; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512GFNI-LABEL: 'splatconstant_rotate_i64' @@ -2934,9 +2934,9 @@ define void @splatconstant_rotate_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256 ; ; XOP-LABEL: 'splatconstant_rotate_i32' ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %a32, i32 5) -; XOP-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> ) -; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> ) -; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512GFNI-LABEL: 'splatconstant_rotate_i32' @@ -2984,9 +2984,9 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; ; AVX512BW-LABEL: 'splatconstant_rotate_i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 3) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512DQ-LABEL: 'splatconstant_rotate_i16' @@ -3019,9 +3019,9 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; ; XOP-LABEL: 'splatconstant_rotate_i16' ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 3) -; XOP-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) -; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) -; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512GFNI-LABEL: 'splatconstant_rotate_i16' @@ -3069,9 +3069,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512BW-LABEL: 'splatconstant_rotate_i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512DQ-LABEL: 'splatconstant_rotate_i8' @@ -3083,9 +3083,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512VBMI2-LABEL: 'splatconstant_rotate_i8' ; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; SLM-LABEL: 'splatconstant_rotate_i8' @@ -3104,16 +3104,16 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; XOP-LABEL: 'splatconstant_rotate_i8' ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) -; XOP-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512GFNI-LABEL: 'splatconstant_rotate_i8' ; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) diff --git a/llvm/test/Analysis/CostModel/X86/fshr.ll b/llvm/test/Analysis/CostModel/X86/fshr.ll index ca9ddcc52938..ada1b9c5bdc4 100644 --- a/llvm/test/Analysis/CostModel/X86/fshr.ll +++ b/llvm/test/Analysis/CostModel/X86/fshr.ll @@ -2682,8 +2682,8 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; ; AVX512BW-LABEL: 'splatconstant_rotate_i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 3) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; @@ -2767,9 +2767,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512BW-LABEL: 'splatconstant_rotate_i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; ; AVX512DQ-LABEL: 'splatconstant_rotate_i8' @@ -2781,9 +2781,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512VBMI2-LABEL: 'splatconstant_rotate_i8' ; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; ; SLM-LABEL: 'splatconstant_rotate_i8' @@ -2809,9 +2809,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512GFNI-LABEL: 'splatconstant_rotate_i8' ; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-fshl-rot.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-fshl-rot.ll index 639aa0a1c6a2..9b8480cd0088 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/arith-fshl-rot.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-fshl-rot.ll @@ -834,22 +834,34 @@ define void @fshl_v2i32_uniformconst() { ; SSE-NEXT: store i32 [[R1]], ptr getelementptr inbounds ([16 x i32], ptr @d32, i32 0, i64 1), align 4 ; SSE-NEXT: ret void ; -; AVX-LABEL: @fshl_v2i32_uniformconst( -; AVX-NEXT: [[A0:%.*]] = load i32, ptr @a32, align 4 -; AVX-NEXT: [[A1:%.*]] = load i32, ptr getelementptr inbounds ([16 x i32], ptr @a32, i32 0, i64 1), align 4 -; AVX-NEXT: [[R0:%.*]] = call i32 @llvm.fshl.i32(i32 [[A0]], i32 [[A0]], i32 1) -; AVX-NEXT: [[R1:%.*]] = call i32 @llvm.fshl.i32(i32 [[A1]], i32 [[A1]], i32 1) -; AVX-NEXT: store i32 [[R0]], ptr @d32, align 4 -; AVX-NEXT: store i32 [[R1]], ptr getelementptr inbounds ([16 x i32], ptr @d32, i32 0, i64 1), align 4 -; AVX-NEXT: ret void +; AVX1-LABEL: @fshl_v2i32_uniformconst( +; AVX1-NEXT: [[A0:%.*]] = load i32, ptr @a32, align 4 +; AVX1-NEXT: [[A1:%.*]] = load i32, ptr getelementptr inbounds ([16 x i32], ptr @a32, i32 0, i64 1), align 4 +; AVX1-NEXT: [[R0:%.*]] = call i32 @llvm.fshl.i32(i32 [[A0]], i32 [[A0]], i32 1) +; AVX1-NEXT: [[R1:%.*]] = call i32 @llvm.fshl.i32(i32 [[A1]], i32 [[A1]], i32 1) +; AVX1-NEXT: store i32 [[R0]], ptr @d32, align 4 +; AVX1-NEXT: store i32 [[R1]], ptr getelementptr inbounds ([16 x i32], ptr @d32, i32 0, i64 1), align 4 +; AVX1-NEXT: ret void +; +; AVX2-LABEL: @fshl_v2i32_uniformconst( +; AVX2-NEXT: [[A0:%.*]] = load i32, ptr @a32, align 4 +; AVX2-NEXT: [[A1:%.*]] = load i32, ptr getelementptr inbounds ([16 x i32], ptr @a32, i32 0, i64 1), align 4 +; AVX2-NEXT: [[R0:%.*]] = call i32 @llvm.fshl.i32(i32 [[A0]], i32 [[A0]], i32 1) +; AVX2-NEXT: [[R1:%.*]] = call i32 @llvm.fshl.i32(i32 [[A1]], i32 [[A1]], i32 1) +; AVX2-NEXT: store i32 [[R0]], ptr @d32, align 4 +; AVX2-NEXT: store i32 [[R1]], ptr getelementptr inbounds ([16 x i32], ptr @d32, i32 0, i64 1), align 4 +; AVX2-NEXT: ret void +; +; AVX256-LABEL: @fshl_v2i32_uniformconst( +; AVX256-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr @a32, align 4 +; AVX256-NEXT: [[TMP2:%.*]] = call <2 x i32> @llvm.fshl.v2i32(<2 x i32> [[TMP1]], <2 x i32> [[TMP1]], <2 x i32> ) +; AVX256-NEXT: store <2 x i32> [[TMP2]], ptr @d32, align 4 +; AVX256-NEXT: ret void ; ; AVX512-LABEL: @fshl_v2i32_uniformconst( -; AVX512-NEXT: [[A0:%.*]] = load i32, ptr @a32, align 4 -; AVX512-NEXT: [[A1:%.*]] = load i32, ptr getelementptr inbounds ([16 x i32], ptr @a32, i32 0, i64 1), align 4 -; AVX512-NEXT: [[R0:%.*]] = call i32 @llvm.fshl.i32(i32 [[A0]], i32 [[A0]], i32 1) -; AVX512-NEXT: [[R1:%.*]] = call i32 @llvm.fshl.i32(i32 [[A1]], i32 [[A1]], i32 1) -; AVX512-NEXT: store i32 [[R0]], ptr @d32, align 4 -; AVX512-NEXT: store i32 [[R1]], ptr getelementptr inbounds ([16 x i32], ptr @d32, i32 0, i64 1), align 4 +; AVX512-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr @a32, align 4 +; AVX512-NEXT: [[TMP2:%.*]] = call <2 x i32> @llvm.fshl.v2i32(<2 x i32> [[TMP1]], <2 x i32> [[TMP1]], <2 x i32> ) +; AVX512-NEXT: store <2 x i32> [[TMP2]], ptr @d32, align 4 ; AVX512-NEXT: ret void ; %a0 = load i32, ptr getelementptr inbounds ([16 x i32], ptr @a32, i32 0, i64 0 ), align 4 diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-fshr-rot.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-fshr-rot.ll index c557c9647551..f3e73d0e6840 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/arith-fshr-rot.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-fshr-rot.ll @@ -834,22 +834,34 @@ define void @fshr_v2i32_uniformconst() { ; SSE-NEXT: store i32 [[R1]], ptr getelementptr inbounds ([16 x i32], ptr @d32, i32 0, i64 1), align 4 ; SSE-NEXT: ret void ; -; AVX-LABEL: @fshr_v2i32_uniformconst( -; AVX-NEXT: [[A0:%.*]] = load i32, ptr @a32, align 4 -; AVX-NEXT: [[A1:%.*]] = load i32, ptr getelementptr inbounds ([16 x i32], ptr @a32, i32 0, i64 1), align 4 -; AVX-NEXT: [[R0:%.*]] = call i32 @llvm.fshr.i32(i32 [[A0]], i32 [[A0]], i32 1) -; AVX-NEXT: [[R1:%.*]] = call i32 @llvm.fshr.i32(i32 [[A1]], i32 [[A1]], i32 1) -; AVX-NEXT: store i32 [[R0]], ptr @d32, align 4 -; AVX-NEXT: store i32 [[R1]], ptr getelementptr inbounds ([16 x i32], ptr @d32, i32 0, i64 1), align 4 -; AVX-NEXT: ret void +; AVX1-LABEL: @fshr_v2i32_uniformconst( +; AVX1-NEXT: [[A0:%.*]] = load i32, ptr @a32, align 4 +; AVX1-NEXT: [[A1:%.*]] = load i32, ptr getelementptr inbounds ([16 x i32], ptr @a32, i32 0, i64 1), align 4 +; AVX1-NEXT: [[R0:%.*]] = call i32 @llvm.fshr.i32(i32 [[A0]], i32 [[A0]], i32 1) +; AVX1-NEXT: [[R1:%.*]] = call i32 @llvm.fshr.i32(i32 [[A1]], i32 [[A1]], i32 1) +; AVX1-NEXT: store i32 [[R0]], ptr @d32, align 4 +; AVX1-NEXT: store i32 [[R1]], ptr getelementptr inbounds ([16 x i32], ptr @d32, i32 0, i64 1), align 4 +; AVX1-NEXT: ret void +; +; AVX2-LABEL: @fshr_v2i32_uniformconst( +; AVX2-NEXT: [[A0:%.*]] = load i32, ptr @a32, align 4 +; AVX2-NEXT: [[A1:%.*]] = load i32, ptr getelementptr inbounds ([16 x i32], ptr @a32, i32 0, i64 1), align 4 +; AVX2-NEXT: [[R0:%.*]] = call i32 @llvm.fshr.i32(i32 [[A0]], i32 [[A0]], i32 1) +; AVX2-NEXT: [[R1:%.*]] = call i32 @llvm.fshr.i32(i32 [[A1]], i32 [[A1]], i32 1) +; AVX2-NEXT: store i32 [[R0]], ptr @d32, align 4 +; AVX2-NEXT: store i32 [[R1]], ptr getelementptr inbounds ([16 x i32], ptr @d32, i32 0, i64 1), align 4 +; AVX2-NEXT: ret void +; +; AVX256-LABEL: @fshr_v2i32_uniformconst( +; AVX256-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr @a32, align 4 +; AVX256-NEXT: [[TMP2:%.*]] = call <2 x i32> @llvm.fshr.v2i32(<2 x i32> [[TMP1]], <2 x i32> [[TMP1]], <2 x i32> ) +; AVX256-NEXT: store <2 x i32> [[TMP2]], ptr @d32, align 4 +; AVX256-NEXT: ret void ; ; AVX512-LABEL: @fshr_v2i32_uniformconst( -; AVX512-NEXT: [[A0:%.*]] = load i32, ptr @a32, align 4 -; AVX512-NEXT: [[A1:%.*]] = load i32, ptr getelementptr inbounds ([16 x i32], ptr @a32, i32 0, i64 1), align 4 -; AVX512-NEXT: [[R0:%.*]] = call i32 @llvm.fshr.i32(i32 [[A0]], i32 [[A0]], i32 1) -; AVX512-NEXT: [[R1:%.*]] = call i32 @llvm.fshr.i32(i32 [[A1]], i32 [[A1]], i32 1) -; AVX512-NEXT: store i32 [[R0]], ptr @d32, align 4 -; AVX512-NEXT: store i32 [[R1]], ptr getelementptr inbounds ([16 x i32], ptr @d32, i32 0, i64 1), align 4 +; AVX512-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr @a32, align 4 +; AVX512-NEXT: [[TMP2:%.*]] = call <2 x i32> @llvm.fshr.v2i32(<2 x i32> [[TMP1]], <2 x i32> [[TMP1]], <2 x i32> ) +; AVX512-NEXT: store <2 x i32> [[TMP2]], ptr @d32, align 4 ; AVX512-NEXT: ret void ; %a0 = load i32, ptr getelementptr inbounds ([16 x i32], ptr @a32, i32 0, i64 0 ), align 4 -- GitLab From 58a08e154c804051aaca9151a8053aea3ec15646 Mon Sep 17 00:00:00 2001 From: Philip Reames Date: Wed, 17 Apr 2024 11:15:25 -0700 Subject: [PATCH 088/862] [RISCV] Add coverage for strength reduction of mul by small negative immediates --- llvm/test/CodeGen/RISCV/rv32zba.ll | 93 ++++++++++++++++++++++++++++++ llvm/test/CodeGen/RISCV/rv64zba.ll | 93 ++++++++++++++++++++++++++++++ 2 files changed, 186 insertions(+) diff --git a/llvm/test/CodeGen/RISCV/rv32zba.ll b/llvm/test/CodeGen/RISCV/rv32zba.ll index cc632a09c805..a78f823d3184 100644 --- a/llvm/test/CodeGen/RISCV/rv32zba.ll +++ b/llvm/test/CodeGen/RISCV/rv32zba.ll @@ -645,3 +645,96 @@ define i32 @addshl_5_8(i32 %a, i32 %b) { %e = add i32 %c, %d ret i32 %e } + +define i32 @mul_neg1(i32 %a) { +; CHECK-LABEL: mul_neg1: +; CHECK: # %bb.0: +; CHECK-NEXT: neg a0, a0 +; CHECK-NEXT: ret + %c = mul i32 %a, -1 + ret i32 %c +} + +define i32 @mul_neg2(i32 %a) { +; CHECK-LABEL: mul_neg2: +; CHECK: # %bb.0: +; CHECK-NEXT: slli a0, a0, 1 +; CHECK-NEXT: neg a0, a0 +; CHECK-NEXT: ret + %c = mul i32 %a, -2 + ret i32 %c +} + +define i32 @mul_neg3(i32 %a) { +; RV32I-LABEL: mul_neg3: +; RV32I: # %bb.0: +; RV32I-NEXT: slli a1, a0, 1 +; RV32I-NEXT: neg a0, a0 +; RV32I-NEXT: sub a0, a0, a1 +; RV32I-NEXT: ret +; +; RV32ZBA-LABEL: mul_neg3: +; RV32ZBA: # %bb.0: +; RV32ZBA-NEXT: sh1add a0, a0, a0 +; RV32ZBA-NEXT: neg a0, a0 +; RV32ZBA-NEXT: ret + %c = mul i32 %a, -3 + ret i32 %c +} + +define i32 @mul_neg4(i32 %a) { +; CHECK-LABEL: mul_neg4: +; CHECK: # %bb.0: +; CHECK-NEXT: slli a0, a0, 2 +; CHECK-NEXT: neg a0, a0 +; CHECK-NEXT: ret + %c = mul i32 %a, -4 + ret i32 %c +} + +define i32 @mul_neg5(i32 %a) { +; RV32I-LABEL: mul_neg5: +; RV32I: # %bb.0: +; RV32I-NEXT: slli a1, a0, 2 +; RV32I-NEXT: neg a0, a0 +; RV32I-NEXT: sub a0, a0, a1 +; RV32I-NEXT: ret +; +; RV32ZBA-LABEL: mul_neg5: +; RV32ZBA: # %bb.0: +; RV32ZBA-NEXT: sh2add a0, a0, a0 +; RV32ZBA-NEXT: neg a0, a0 +; RV32ZBA-NEXT: ret + %c = mul i32 %a, -5 + ret i32 %c +} + +define i32 @mul_neg6(i32 %a) { +; CHECK-LABEL: mul_neg6: +; CHECK: # %bb.0: +; CHECK-NEXT: li a1, -6 +; CHECK-NEXT: mul a0, a0, a1 +; CHECK-NEXT: ret + %c = mul i32 %a, -6 + ret i32 %c +} + +define i32 @mul_neg7(i32 %a) { +; CHECK-LABEL: mul_neg7: +; CHECK: # %bb.0: +; CHECK-NEXT: slli a1, a0, 3 +; CHECK-NEXT: sub a0, a0, a1 +; CHECK-NEXT: ret + %c = mul i32 %a, -7 + ret i32 %c +} + +define i32 @mul_neg8(i32 %a) { +; CHECK-LABEL: mul_neg8: +; CHECK: # %bb.0: +; CHECK-NEXT: slli a0, a0, 3 +; CHECK-NEXT: neg a0, a0 +; CHECK-NEXT: ret + %c = mul i32 %a, -8 + ret i32 %c +} diff --git a/llvm/test/CodeGen/RISCV/rv64zba.ll b/llvm/test/CodeGen/RISCV/rv64zba.ll index b4c80b60e0ba..6939185947f4 100644 --- a/llvm/test/CodeGen/RISCV/rv64zba.ll +++ b/llvm/test/CodeGen/RISCV/rv64zba.ll @@ -2533,3 +2533,96 @@ define i64 @regression(i32 signext %x, i32 signext %y) { %res = mul nuw nsw i64 %ext, 24 ret i64 %res } + +define i64 @mul_neg1(i64 %a) { +; CHECK-LABEL: mul_neg1: +; CHECK: # %bb.0: +; CHECK-NEXT: neg a0, a0 +; CHECK-NEXT: ret + %c = mul i64 %a, -1 + ret i64 %c +} + +define i64 @mul_neg2(i64 %a) { +; CHECK-LABEL: mul_neg2: +; CHECK: # %bb.0: +; CHECK-NEXT: slli a0, a0, 1 +; CHECK-NEXT: neg a0, a0 +; CHECK-NEXT: ret + %c = mul i64 %a, -2 + ret i64 %c +} + +define i64 @mul_neg3(i64 %a) { +; RV64I-LABEL: mul_neg3: +; RV64I: # %bb.0: +; RV64I-NEXT: slli a1, a0, 1 +; RV64I-NEXT: neg a0, a0 +; RV64I-NEXT: sub a0, a0, a1 +; RV64I-NEXT: ret +; +; RV64ZBA-LABEL: mul_neg3: +; RV64ZBA: # %bb.0: +; RV64ZBA-NEXT: sh1add a0, a0, a0 +; RV64ZBA-NEXT: neg a0, a0 +; RV64ZBA-NEXT: ret + %c = mul i64 %a, -3 + ret i64 %c +} + +define i64 @mul_neg4(i64 %a) { +; CHECK-LABEL: mul_neg4: +; CHECK: # %bb.0: +; CHECK-NEXT: slli a0, a0, 2 +; CHECK-NEXT: neg a0, a0 +; CHECK-NEXT: ret + %c = mul i64 %a, -4 + ret i64 %c +} + +define i64 @mul_neg5(i64 %a) { +; RV64I-LABEL: mul_neg5: +; RV64I: # %bb.0: +; RV64I-NEXT: slli a1, a0, 2 +; RV64I-NEXT: neg a0, a0 +; RV64I-NEXT: sub a0, a0, a1 +; RV64I-NEXT: ret +; +; RV64ZBA-LABEL: mul_neg5: +; RV64ZBA: # %bb.0: +; RV64ZBA-NEXT: sh2add a0, a0, a0 +; RV64ZBA-NEXT: neg a0, a0 +; RV64ZBA-NEXT: ret + %c = mul i64 %a, -5 + ret i64 %c +} + +define i64 @mul_neg6(i64 %a) { +; CHECK-LABEL: mul_neg6: +; CHECK: # %bb.0: +; CHECK-NEXT: li a1, -6 +; CHECK-NEXT: mul a0, a0, a1 +; CHECK-NEXT: ret + %c = mul i64 %a, -6 + ret i64 %c +} + +define i64 @mul_neg7(i64 %a) { +; CHECK-LABEL: mul_neg7: +; CHECK: # %bb.0: +; CHECK-NEXT: slli a1, a0, 3 +; CHECK-NEXT: sub a0, a0, a1 +; CHECK-NEXT: ret + %c = mul i64 %a, -7 + ret i64 %c +} + +define i64 @mul_neg8(i64 %a) { +; CHECK-LABEL: mul_neg8: +; CHECK: # %bb.0: +; CHECK-NEXT: slli a0, a0, 3 +; CHECK-NEXT: neg a0, a0 +; CHECK-NEXT: ret + %c = mul i64 %a, -8 + ret i64 %c +} -- GitLab From cc82f1290a1e2157a6c0530d78d8cc84d2b8553d Mon Sep 17 00:00:00 2001 From: Usman Nadeem Date: Wed, 17 Apr 2024 11:42:52 -0700 Subject: [PATCH 089/862] [AArch64] Update latencies for Cortex-A510 scheduling model (#87293) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updated according to the Software Optimization Guide for Arm® Cortex®‑A510 Core Revision: r1p3 Issue 6.0. --- llvm/lib/Target/AArch64/AArch64SchedA510.td | 145 +- .../AArch64/GlobalISel/combine-udiv.ll | 8 +- llvm/test/CodeGen/AArch64/aarch64-addv.ll | 10 +- .../AArch64/aarch64-dup-ext-scalable.ll | 40 +- llvm/test/CodeGen/AArch64/aarch64-smull.ll | 2 +- llvm/test/CodeGen/AArch64/active_lane_mask.ll | 169 +- .../CodeGen/AArch64/arm64-convert-v4f64.ll | 14 +- llvm/test/CodeGen/AArch64/arm64-vabs.ll | 106 +- llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll | 28 +- llvm/test/CodeGen/AArch64/arm64-vhadd.ll | 8 +- ...interleaving-add-mull-scalable-contract.ll | 64 +- ...x-deinterleaving-add-mull-scalable-fast.ll | 63 +- ...complex-deinterleaving-f16-mul-scalable.ll | 23 +- ...complex-deinterleaving-f32-mul-scalable.ll | 18 +- ...complex-deinterleaving-f64-mul-scalable.ll | 18 +- ...complex-deinterleaving-i16-mul-scalable.ll | 3 +- ...rleaving-reductions-predicated-scalable.ll | 64 +- ...plex-deinterleaving-reductions-scalable.ll | 26 +- .../complex-deinterleaving-splat-scalable.ll | 16 +- .../AArch64/concat_vector-truncate-combine.ll | 2 +- .../AArch64/dag-combine-concat-vectors.ll | 12 +- .../div-rem-pair-recomposition-signed.ll | 6 +- .../div-rem-pair-recomposition-unsigned.ll | 6 +- llvm/test/CodeGen/AArch64/extbinopload.ll | 121 +- llvm/test/CodeGen/AArch64/fcmp.ll | 108 +- llvm/test/CodeGen/AArch64/fdiv-combine.ll | 2 +- .../fold-int-pow2-with-fmul-or-fdiv.ll | 2 +- .../CodeGen/AArch64/fp-veclib-expansion.ll | 8 +- llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll | 8 +- .../test/CodeGen/AArch64/fptosi-sat-vector.ll | 20 +- llvm/test/CodeGen/AArch64/funnel-shift-rot.ll | 4 +- ...st-and-by-const-from-lshr-in-eqcmp-zero.ll | 2 +- llvm/test/CodeGen/AArch64/icmp.ll | 4 +- llvm/test/CodeGen/AArch64/insert-extend.ll | 54 +- .../insert-subvector-res-legalization.ll | 70 +- .../AArch64/intrinsic-cttz-elts-sve.ll | 91 +- llvm/test/CodeGen/AArch64/itofp.ll | 1132 ++++++------ llvm/test/CodeGen/AArch64/ldexp.ll | 6 +- .../CodeGen/AArch64/llvm-ir-to-intrinsic.ll | 43 +- llvm/test/CodeGen/AArch64/load-insert-zero.ll | 28 +- llvm/test/CodeGen/AArch64/logic-shift.ll | 48 +- .../AArch64/named-vector-shuffles-sve.ll | 76 +- llvm/test/CodeGen/AArch64/neon-dotreduce.ll | 758 ++++---- llvm/test/CodeGen/AArch64/neon-extadd.ll | 12 +- llvm/test/CodeGen/AArch64/neon-shift-neg.ll | 24 +- .../CodeGen/AArch64/predicated-add-sub.ll | 22 +- .../AArch64/ragreedy-local-interval-cost.ll | 262 ++- llvm/test/CodeGen/AArch64/rcpc3-sve.ll | 4 +- llvm/test/CodeGen/AArch64/reassocmls.ll | 6 +- llvm/test/CodeGen/AArch64/reduce-shuffle.ll | 356 ++-- llvm/test/CodeGen/AArch64/sat-add.ll | 8 +- llvm/test/CodeGen/AArch64/sext.ll | 36 +- .../CodeGen/AArch64/sink-addsub-of-const.ll | 24 +- ...ate-sm-changing-call-disable-coalescing.ll | 130 +- .../sme-streaming-compatible-interface.ll | 2 +- .../AArch64/sme2-intrinsics-fp-dots.ll | 12 +- .../AArch64/sme2-intrinsics-int-dots.ll | 42 +- .../CodeGen/AArch64/sme2-intrinsics-max.ll | 305 ++- .../CodeGen/AArch64/sme2-intrinsics-min.ll | 163 +- .../CodeGen/AArch64/sme2-intrinsics-mlall.ll | 54 +- .../CodeGen/AArch64/sme2-intrinsics-rshl.ll | 161 +- .../AArch64/sme2-intrinsics-sqdmulh.ll | 89 +- .../CodeGen/AArch64/split-vector-insert.ll | 265 ++- .../CodeGen/AArch64/srem-seteq-vec-splat.ll | 16 +- llvm/test/CodeGen/AArch64/srem-vector-lkk.ll | 4 +- llvm/test/CodeGen/AArch64/sve-abd.ll | 18 +- llvm/test/CodeGen/AArch64/sve-bitcast.ll | 20 +- .../AArch64/sve-calling-convention-mixed.ll | 116 +- llvm/test/CodeGen/AArch64/sve-cmp-folds.ll | 4 +- llvm/test/CodeGen/AArch64/sve-doublereduct.ll | 8 +- llvm/test/CodeGen/AArch64/sve-expand-div.ll | 16 +- .../CodeGen/AArch64/sve-extract-element.ll | 10 +- .../sve-extract-fixed-from-scalable-vector.ll | 8 +- .../AArch64/sve-extract-fixed-vector.ll | 16 +- .../AArch64/sve-extract-scalable-vector.ll | 18 +- llvm/test/CodeGen/AArch64/sve-fcmp.ll | 2 +- llvm/test/CodeGen/AArch64/sve-fcopysign.ll | 18 +- llvm/test/CodeGen/AArch64/sve-fcvt.ll | 64 +- .../sve-fixed-length-addressing-modes.ll | 4 +- .../AArch64/sve-fixed-length-build-vector.ll | 13 +- .../AArch64/sve-fixed-length-concat.ll | 112 +- .../sve-fixed-length-extract-vector-elt.ll | 24 +- .../AArch64/sve-fixed-length-fcopysign.ll | 16 +- .../sve-fixed-length-fp-extend-trunc.ll | 6 +- .../AArch64/sve-fixed-length-fp-select.ll | 234 +-- .../AArch64/sve-fixed-length-fp-to-int.ll | 24 +- .../CodeGen/AArch64/sve-fixed-length-fp128.ll | 8 +- .../sve-fixed-length-frame-offests-crash.ll | 32 +- .../sve-fixed-length-insert-vector-elt.ll | 222 +-- .../AArch64/sve-fixed-length-int-arith.ll | 4 +- .../AArch64/sve-fixed-length-int-div.ll | 56 +- .../AArch64/sve-fixed-length-int-extends.ll | 12 +- .../AArch64/sve-fixed-length-int-rem.ll | 88 +- .../AArch64/sve-fixed-length-int-select.ll | 312 ++-- .../AArch64/sve-fixed-length-int-to-fp.ll | 24 +- .../AArch64/sve-fixed-length-mask-opt.ll | 6 +- .../sve-fixed-length-masked-128bit-loads.ll | 2 +- .../sve-fixed-length-masked-128bit-stores.ll | 18 +- .../AArch64/sve-fixed-length-masked-gather.ll | 148 +- .../AArch64/sve-fixed-length-masked-loads.ll | 32 +- .../sve-fixed-length-masked-scatter.ll | 156 +- .../AArch64/sve-fixed-length-masked-stores.ll | 42 +- .../AArch64/sve-fixed-length-shuffles.ll | 26 +- .../AArch64/sve-fixed-length-splat-vector.ll | 84 +- .../AArch64/sve-fixed-length-trunc-stores.ll | 16 +- .../sve-fixed-length-vector-shuffle-tbl.ll | 6 +- .../CodeGen/AArch64/sve-fp-int-min-max.ll | 2 +- .../test/CodeGen/AArch64/sve-fp-reciprocal.ll | 6 +- .../CodeGen/AArch64/sve-fp-reduce-fadda.ll | 4 +- llvm/test/CodeGen/AArch64/sve-fptosi-sat.ll | 282 +-- llvm/test/CodeGen/AArch64/sve-fptoui-sat.ll | 32 +- .../AArch64/sve-gather-scatter-addr-opts.ll | 28 +- llvm/test/CodeGen/AArch64/sve-hadd.ll | 20 +- .../AArch64/sve-implicit-zero-filling.ll | 8 +- .../CodeGen/AArch64/sve-insert-element.ll | 88 +- .../test/CodeGen/AArch64/sve-insert-vector.ll | 22 +- .../test/CodeGen/AArch64/sve-int-arith-imm.ll | 34 +- llvm/test/CodeGen/AArch64/sve-int-arith.ll | 20 +- llvm/test/CodeGen/AArch64/sve-int-reduce.ll | 62 +- .../CodeGen/AArch64/sve-intrinsics-index.ll | 2 +- .../AArch64/sve-intrinsics-int-arith-imm.ll | 38 +- .../AArch64/sve-intrinsics-logical-imm.ll | 2 +- llvm/test/CodeGen/AArch64/sve-ld-post-inc.ll | 30 +- llvm/test/CodeGen/AArch64/sve-ld1r.ll | 16 +- .../sve-lsr-scaled-index-addressing-mode.ll | 4 +- .../AArch64/sve-masked-gather-legalize.ll | 8 +- .../CodeGen/AArch64/sve-masked-ldst-sext.ll | 2 +- .../CodeGen/AArch64/sve-masked-ldst-zext.ll | 6 +- .../AArch64/sve-masked-scatter-legalize.ll | 20 +- .../CodeGen/AArch64/sve-masked-scatter.ll | 2 +- llvm/test/CodeGen/AArch64/sve-pr62151.ll | 2 +- llvm/test/CodeGen/AArch64/sve-pred-arith.ll | 40 +- .../test/CodeGen/AArch64/sve-pred-selectop.ll | 108 +- .../CodeGen/AArch64/sve-pred-selectop2.ll | 76 +- .../CodeGen/AArch64/sve-pred-selectop3.ll | 28 +- .../AArch64/sve-ptest-removal-cmple.ll | 8 +- .../CodeGen/AArch64/sve-redundant-store.ll | 2 +- .../CodeGen/AArch64/sve-split-extract-elt.ll | 42 +- llvm/test/CodeGen/AArch64/sve-split-fcvt.ll | 26 +- .../CodeGen/AArch64/sve-split-fp-reduce.ll | 2 +- .../CodeGen/AArch64/sve-split-insert-elt.ll | 26 +- .../CodeGen/AArch64/sve-split-int-reduce.ll | 8 +- llvm/test/CodeGen/AArch64/sve-split-load.ll | 4 +- llvm/test/CodeGen/AArch64/sve-split-store.ll | 8 +- .../CodeGen/AArch64/sve-srem-combine-loop.ll | 2 +- .../sve-st1-addressing-mode-reg-imm.ll | 8 +- llvm/test/CodeGen/AArch64/sve-stepvector.ll | 2 +- ...treaming-mode-fixed-length-bit-counting.ll | 36 +- ...e-streaming-mode-fixed-length-ext-loads.ll | 10 +- ...e-streaming-mode-fixed-length-fcopysign.ll | 26 +- ...ve-streaming-mode-fixed-length-fp-arith.ll | 48 +- ...streaming-mode-fixed-length-fp-compares.ll | 44 +- ...-streaming-mode-fixed-length-fp-convert.ll | 4 +- ...aming-mode-fixed-length-fp-extend-trunc.ll | 10 +- .../sve-streaming-mode-fixed-length-fp-fma.ll | 6 +- ...e-streaming-mode-fixed-length-fp-minmax.ll | 24 +- ...e-streaming-mode-fixed-length-fp-reduce.ll | 30 +- ...streaming-mode-fixed-length-fp-rounding.ll | 42 +- ...e-streaming-mode-fixed-length-fp-select.ll | 18 +- ...e-streaming-mode-fixed-length-fp-to-int.ll | 118 +- ...-streaming-mode-fixed-length-fp-vselect.ll | 6 +- ...ing-mode-fixed-length-insert-vector-elt.ll | 42 +- ...e-streaming-mode-fixed-length-int-arith.ll | 16 +- ...treaming-mode-fixed-length-int-compares.ll | 14 +- ...sve-streaming-mode-fixed-length-int-div.ll | 34 +- ...eaming-mode-fixed-length-int-immediates.ll | 8 +- ...-streaming-mode-fixed-length-int-minmax.ll | 32 +- ...ve-streaming-mode-fixed-length-int-mulh.ll | 20 +- ...-streaming-mode-fixed-length-int-reduce.ll | 40 +- ...sve-streaming-mode-fixed-length-int-rem.ll | 64 +- ...-streaming-mode-fixed-length-int-select.ll | 28 +- ...-streaming-mode-fixed-length-int-shifts.ll | 32 +- ...e-streaming-mode-fixed-length-int-to-fp.ll | 56 +- ...streaming-mode-fixed-length-int-vselect.ll | 10 +- ...-streaming-mode-fixed-length-log-reduce.ll | 24 +- ...treaming-mode-fixed-length-masked-store.ll | 16 +- ...eaming-mode-fixed-length-optimize-ptrue.ll | 10 +- ...streaming-mode-fixed-length-permute-rev.ll | 24 +- ...g-mode-fixed-length-permute-zip-uzp-trn.ll | 30 +- .../sve-streaming-mode-fixed-length-ptest.ll | 78 +- .../sve-streaming-mode-fixed-length-rev.ll | 14 +- ...e-streaming-mode-fixed-length-sdiv-pow2.ll | 8 +- ...sve-streaming-mode-fixed-length-shuffle.ll | 2 +- .../sve-streaming-mode-fixed-length-stores.ll | 4 +- .../sve-streaming-mode-fixed-length-trunc.ll | 354 ++-- llvm/test/CodeGen/AArch64/sve-trunc.ll | 36 +- llvm/test/CodeGen/AArch64/sve-umulo-sdnode.ll | 20 +- .../sve-uunpklo-load-uzp1-store-combine.ll | 8 +- .../test/CodeGen/AArch64/sve-vecreduce-dot.ll | 2 +- .../CodeGen/AArch64/sve-vecreduce-fold.ll | 2 +- llvm/test/CodeGen/AArch64/sve2-fcopysign.ll | 16 +- .../AArch64/sve2-fixed-length-fcopysign.ll | 16 +- .../AArch64/sve2-intrinsics-combine-rshrnb.ll | 22 +- .../AArch64/sve2-intrinsics-int-arith-imm.ll | 2 +- llvm/test/CodeGen/AArch64/sve2-rsh.ll | 2 +- llvm/test/CodeGen/AArch64/sve2-xar.ll | 2 +- .../AArch64/sve2p1-intrinsics-selx2.ll | 32 +- .../AArch64/sve2p1-intrinsics-selx4.ll | 64 +- .../AArch64/sve2p1-intrinsics-stores.ll | 64 +- llvm/test/CodeGen/AArch64/uadd_sat_vec.ll | 10 +- .../CodeGen/AArch64/urem-seteq-vec-nonzero.ll | 10 +- llvm/test/CodeGen/AArch64/vec_uaddo.ll | 8 +- llvm/test/CodeGen/AArch64/vecreduce-add.ll | 546 +++--- llvm/test/CodeGen/AArch64/vector-fcopysign.ll | 46 +- llvm/test/CodeGen/AArch64/vector-gep.ll | 4 +- .../test/CodeGen/AArch64/vselect-constants.ll | 5 +- llvm/test/CodeGen/AArch64/zext-to-tbl.ll | 50 +- .../AArch64/Cortex/A510-neon-instructions.s | 376 ++-- .../AArch64/Cortex/A510-sve-instructions.s | 1636 ++++++++--------- 209 files changed, 6488 insertions(+), 6481 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64SchedA510.td b/llvm/lib/Target/AArch64/AArch64SchedA510.td index 68343674bc81..945687894615 100644 --- a/llvm/lib/Target/AArch64/AArch64SchedA510.td +++ b/llvm/lib/Target/AArch64/AArch64SchedA510.td @@ -254,7 +254,7 @@ def : InstRW<[WriteIS], (instrs RBITWr, RBITXr)>; // Compute pointer authentication code for data address // Compute pointer authentication code, using generic key // Compute pointer authentication code for instruction address -def : InstRW<[CortexA510Write<3, CortexA510UnitPAC>], (instregex "^AUT", "^PAC")>; +def : InstRW<[CortexA510Write<5, CortexA510UnitPAC>], (instregex "^AUT", "^PAC")>; // Branch and link, register, with pointer authentication // Branch, register, with pointer authentication @@ -401,30 +401,30 @@ def : InstRW<[CortexA510WriteFPALU_F3], (instrs FCSELHrrr, FCSELSrrr, FCSELDrrr) def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "[SU]ABDv(2i32|4i16|8i8)")>; def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "[SU]ABDv(16i8|4i32|8i16)")>; // ASIMD absolute diff accum -def : InstRW<[CortexA510Write<8, CortexA510UnitVALU>], (instregex "[SU]ABAL?v")>; +def : InstRW<[CortexA510Write<6, CortexA510UnitVALU>], (instregex "[SU]ABAL?v")>; // ASIMD absolute diff long def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "[SU]ABDLv")>; // ASIMD arith #1 -def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "(ADD|SUB|NEG)v(1i64|2i32|4i16|8i8)", - "[SU]R?HADDv(2i32|4i16|8i8)", "[SU]HSUBv(2i32|4i16|8i8)")>; -def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "(ADD|SUB|NEG)v(2i64|4i32|8i16|16i8)", - "[SU]R?HADDv(8i16|4i32|16i8)", "[SU]HSUBv(8i16|4i32|16i8)")>; +def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "(ADD|SUB|NEG)v", + "[SU]R?HADDv", "[SU]HSUBv")>; // ASIMD arith #2 -def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "ABSv(1i64|2i32|4i16|8i8)$", +def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "ABSv(1i64|2i32|4i16|8i8)$", "[SU]ADDLPv(2i32_v1i64|4i16_v2i32|8i8_v4i16)$", - "([SU]QADD|[SU]QSUB|SQNEG|SUQADD|USQADD)v(1i16|1i32|1i64|1i8|2i32|4i16|8i8)$", "ADDPv(2i32|4i16|8i8)$")>; -def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "ABSv(2i64|4i32|8i16|16i8)$", +def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "([SU]QADD|[SU]QSUB|SQNEG|SUQADD|USQADD)v(1i16|1i32|1i64|1i8|2i32|4i16|8i8)$")>; +def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "ABSv(2i64|4i32|8i16|16i8)$", "[SU]ADDLPv(16i8_v8i16|4i32_v2i64|8i16_v4i32)$", - "([SU]QADD|[SU]QSUB|SQNEG|SUQADD|USQADD)v(16i8|2i64|4i32|8i16)$", "ADDPv(16i8|2i64|4i32|8i16)$")>; +def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "([SU]QADD|[SU]QSUB|SQNEG|SUQADD|USQADD)v(16i8|2i64|4i32|8i16)$")>; // ASIMD arith #3 -def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "SADDLv", "UADDLv", "SADDWv", - "UADDWv", "SSUBLv", "USUBLv", "SSUBWv", "USUBWv", "ADDHNv", "SUBHNv")>; +def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "SADDLv", "UADDLv", "SADDWv", + "UADDWv", "SSUBLv", "USUBLv", "SSUBWv", "USUBWv")>; +def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "ADDHNv", "SUBHNv")>; // ASIMD arith #5 -def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "RADDHNv", "RSUBHNv")>; +def : InstRW<[CortexA510Write<8, CortexA510UnitVALU>], (instregex "RADDHNv", "RSUBHNv")>; // ASIMD arith, reduce -def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "ADDVv", "SADDLVv", "UADDLVv")>; +def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "ADDVv")>; +def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "SADDLVv", "UADDLVv")>; // ASIMD compare #1 def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "CM(EQ|GE|GT|HI|HS|LE|LT)v(1i64|2i32|4i16|8i8)")>; def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "CM(EQ|GE|GT|HI|HS|LE|LT)v(2i64|4i32|8i16|16i8)")>; @@ -437,8 +437,8 @@ def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "(AND|EOR|NOT| def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "(AND|EOR|NOT|ORN)v16i8", "(ORR|BIC)v(16i8|4i32|8i16)$", "MVNIv(4i32|4s|8i16)")>; // ASIMD max/min, basic -def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "[SU](MIN|MAX)P?v(2i32|4i16|8i8)")>; -def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "[SU](MIN|MAX)P?v(16i8|4i132|8i16)")>; +def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "[SU](MIN|MAX)P?v(2i32|4i16|8i8)")>; +def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "[SU](MIN|MAX)P?v(16i8|4i132|8i16)")>; // SIMD max/min, reduce def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "[SU](MAX|MIN)Vv")>; // ASIMD multiply, by element @@ -467,12 +467,12 @@ def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "[SU]MULLv", " // ASIMD polynomial (8x8) multiply long def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instrs PMULLv8i8, PMULLv16i8)>; // ASIMD pairwise add and accumulate -def : InstRW<[CortexA510MCWrite<8, 2, CortexA510UnitVALU>], (instregex "[SU]ADALPv")>; +def : InstRW<[CortexA510MCWrite<7, 2, CortexA510UnitVALU>], (instregex "[SU]ADALPv")>; // ASIMD shift accumulate -def : InstRW<[CortexA510MCWrite<8, 2, CortexA510UnitVALU>], (instregex "[SU]SRA(d|v2i32|v4i16|v8i8)")>; -def : InstRW<[CortexA510MCWrite<8, 2, CortexA510UnitVALU>], (instregex "[SU]SRAv(16i8|2i64|4i32|8i16)")>; +def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "[SU]SRA(d|v2i32|v4i16|v8i8)")>; +def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "[SU]SRAv(16i8|2i64|4i32|8i16)")>; // ASIMD shift accumulate #2 -def : InstRW<[CortexA510MCWrite<8, 2, CortexA510UnitVALU>], (instregex "[SU]RSRA[vd]")>; +def : InstRW<[CortexA510MCWrite<7, 2, CortexA510UnitVALU>], (instregex "[SU]RSRA[vd]")>; // ASIMD shift by immed def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "SHLd$", "SHLv", "SLId$", "SRId$", "[SU]SHR[vd]", "SHRNv")>; @@ -504,7 +504,7 @@ def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "[SU]QRSHLv(2i def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^AES[DE]rr$", "^AESI?MCrr")>; // Crypto polynomial (64x64) multiply long -def : InstRW<[CortexA510MCWrite<8, 0, CortexA510UnitVMC>], (instrs PMULLv1i64, PMULLv2i64)>; +def : InstRW<[CortexA510MCWrite<4, 0, CortexA510UnitVMC>], (instrs PMULLv1i64, PMULLv2i64)>; // Crypto SHA1 hash acceleration op // Crypto SHA1 schedule acceleration ops @@ -512,25 +512,26 @@ def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^SHA1(H|SU0|S // Crypto SHA1 hash acceleration ops // Crypto SHA256 hash acceleration ops -def : InstRW<[CortexA510MCWrite<8, 0, CortexA510UnitVMC>], (instregex "^SHA1[CMP]", "^SHA256H2?")>; +def : InstRW<[CortexA510MCWrite<4, 0, CortexA510UnitVMC>], (instregex "^SHA1[CMP]", "^SHA256H2?")>; // Crypto SHA256 schedule acceleration ops -def : InstRW<[CortexA510MCWrite<8, 0, CortexA510UnitVMC>], (instregex "^SHA256SU[01]")>; +def : InstRW<[CortexA510MCWrite<4, 0, CortexA510UnitVMC>], (instregex "^SHA256SU[01]")>; // Crypto SHA512 hash acceleration ops -def : InstRW<[CortexA510MCWrite<8, 0, CortexA510UnitVMC>], (instregex "^SHA512(H|H2|SU0|SU1)")>; +def : InstRW<[CortexA510MCWrite<9, 0, CortexA510UnitVMC>], (instregex "^SHA512(H|H2|SU0|SU1)")>; // Crypto SHA3 ops -def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instrs BCAX, EOR3, XAR)>; -def : InstRW<[CortexA510MCWrite<8, 0, CortexA510UnitVMC>], (instrs RAX1)>; +def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instrs BCAX, EOR3)>; +def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instrs XAR)>; +def : InstRW<[CortexA510MCWrite<9, 0, CortexA510UnitVMC>], (instrs RAX1)>; // Crypto SM3 ops -def : InstRW<[CortexA510MCWrite<8, 0, CortexA510UnitVMC>], (instregex "^SM3PARTW[12]$", "^SM3SS1$", +def : InstRW<[CortexA510MCWrite<9, 0, CortexA510UnitVMC>], (instregex "^SM3PARTW[12]$", "^SM3SS1$", "^SM3TT[12][AB]$")>; // Crypto SM4 ops -def : InstRW<[CortexA510MCWrite<8, 0, CortexA510UnitVMC>], (instrs SM4E, SM4ENCKEY)>; +def : InstRW<[CortexA510MCWrite<9, 0, CortexA510UnitVMC>], (instrs SM4E, SM4ENCKEY)>; // CRC // ----------------------------------------------------------------------------- @@ -540,25 +541,25 @@ def : InstRW<[CortexA510MCWrite<2, 0, CortexA510UnitMAC>], (instregex "^CRC32")> // SVE Predicate instructions // Loop control, based on predicate -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instrs BRKA_PPmP, BRKA_PPzP, +def : InstRW<[CortexA510Write<2, CortexA510UnitVALU0>], (instrs BRKA_PPmP, BRKA_PPzP, BRKB_PPmP, BRKB_PPzP)>; // Loop control, based on predicate and flag setting -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instrs BRKAS_PPzP, BRKBS_PPzP)>; +def : InstRW<[CortexA510Write<2, CortexA510UnitVALU0>], (instrs BRKAS_PPzP, BRKBS_PPzP)>; // Loop control, propagating -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instrs BRKN_PPzP, BRKPA_PPzPP, BRKPB_PPzPP)>; +def : InstRW<[CortexA510Write<2, CortexA510UnitVALU0>], (instrs BRKN_PPzP, BRKPA_PPzPP, BRKPB_PPzPP)>; // Loop control, propagating and flag setting -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instrs BRKNS_PPzP)>; -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instrs BRKPAS_PPzPP, BRKPBS_PPzPP)>; +def : InstRW<[CortexA510Write<2, CortexA510UnitVALU0>], (instrs BRKNS_PPzP)>; +def : InstRW<[CortexA510Write<4, CortexA510UnitVALU0>], (instrs BRKPAS_PPzPP, BRKPBS_PPzPP)>; // Loop control, based on GPR -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], +def : InstRW<[CortexA510Write<2, CortexA510UnitVALU0>], (instregex "^WHILE(GE|GT|HI|HS|LE|LO|LS|LT)_P(WW|XX)_[BHSD]")>; -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instregex "^WHILE(RW|WR)_PXX_[BHSD]")>; +def : InstRW<[CortexA510Write<2, CortexA510UnitVALU0>], (instregex "^WHILE(RW|WR)_PXX_[BHSD]")>; // Loop terminate def : InstRW<[CortexA510Write<1, CortexA510UnitALU>], (instregex "^CTERM(EQ|NE)_(WW|XX)")>; @@ -569,20 +570,20 @@ def : InstRW<[CortexA510Write<1, CortexA510UnitALU>], (instrs ADDPL_XXI, ADDVL_X def : InstRW<[CortexA510Write<1, CortexA510UnitALU>], (instregex "^CNT[BHWD]_XPiI")>; -def : InstRW<[CortexA510Write<1, CortexA510UnitALU>], +def : InstRW<[CortexA510Write<3, CortexA510UnitALU>], (instregex "^(INC|DEC)[BHWD]_XPiI")>; -def : InstRW<[CortexA510Write<1, CortexA510UnitALU>], +def : InstRW<[CortexA510Write<4, CortexA510UnitALU>], (instregex "^(SQINC|SQDEC|UQINC|UQDEC)[BHWD]_[XW]Pi(Wd)?I")>; // Predicate counting scalar, active predicate -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], +def : InstRW<[CortexA510Write<4, CortexA510UnitVALU0>], (instregex "^CNTP_XPP_[BHSD]")>; -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], +def : InstRW<[CortexA510Write<4, CortexA510UnitVALU0>], (instregex "^(DEC|INC)P_XP_[BHSD]")>; -def : InstRW<[CortexA510Write<8, CortexA510UnitVALU0>], +def : InstRW<[CortexA510Write<9, CortexA510UnitVALU0>], (instregex "^(SQDEC|SQINC|UQDEC|UQINC)P_XP_[BHSD]", "^(UQDEC|UQINC)P_WP_[BHSD]", "^(SQDEC|SQINC|UQDEC|UQINC)P_XPWd_[BHSD]")>; @@ -593,39 +594,39 @@ def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^(DEC|INC|SQDEC|SQINC|UQDEC|UQINC)P_ZP_[HSD]")>; // Predicate logical -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], +def : InstRW<[CortexA510Write<2, CortexA510UnitVALU0>], (instregex "^(AND|BIC|EOR|NAND|NOR|ORN|ORR)_PPzPP")>; // Predicate logical, flag setting -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], +def : InstRW<[CortexA510Write<2, CortexA510UnitVALU0>], (instregex "^(ANDS|BICS|EORS|NANDS|NORS|ORNS|ORRS)_PPzPP")>; // Predicate reverse -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instregex "^REV_PP_[BHSD]")>; +def : InstRW<[CortexA510Write<2, CortexA510UnitVALU0>], (instregex "^REV_PP_[BHSD]")>; // Predicate select -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instrs SEL_PPPP)>; +def : InstRW<[CortexA510Write<2, CortexA510UnitVALU0>], (instrs SEL_PPPP)>; // Predicate set -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instregex "^PFALSE", "^PTRUE_[BHSD]")>; +def : InstRW<[CortexA510Write<2, CortexA510UnitVALU0>], (instregex "^PFALSE", "^PTRUE_[BHSD]")>; // Predicate set/initialize, set flags -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instregex "^PTRUES_[BHSD]")>; +def : InstRW<[CortexA510Write<2, CortexA510UnitVALU0>], (instregex "^PTRUES_[BHSD]")>; // Predicate find first/next -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instregex "^PFIRST_B", "^PNEXT_[BHSD]")>; +def : InstRW<[CortexA510Write<2, CortexA510UnitVALU0>], (instregex "^PFIRST_B", "^PNEXT_[BHSD]")>; // Predicate test -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instrs PTEST_PP)>; +def : InstRW<[CortexA510Write<2, CortexA510UnitVALU0>], (instrs PTEST_PP)>; // Predicate transpose -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instregex "^TRN[12]_PPP_[BHSDQ]")>; +def : InstRW<[CortexA510Write<2, CortexA510UnitVALU0>], (instregex "^TRN[12]_PPP_[BHSDQ]")>; // Predicate unpack and widen -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instrs PUNPKHI_PP, PUNPKLO_PP)>; +def : InstRW<[CortexA510Write<2, CortexA510UnitVALU0>], (instrs PUNPKHI_PP, PUNPKLO_PP)>; // Predicate zip/unzip -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instregex "^(ZIP|UZP)[12]_PPP_[BHSDQ]")>; +def : InstRW<[CortexA510Write<2, CortexA510UnitVALU0>], (instregex "^(ZIP|UZP)[12]_PPP_[BHSDQ]")>; // SVE integer instructions @@ -634,10 +635,10 @@ def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instregex "^(ZIP|UZP)[1 def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^[SU]ABD_(ZPmZ|ZPZZ)_[BHSD]")>; // Arithmetic, absolute diff accum -def : InstRW<[CortexA510MCWrite<8, 2, CortexA510UnitVALU>], (instregex "^[SU]ABA_ZZZ_[BHSD]")>; +def : InstRW<[CortexA510MCWrite<6, 2, CortexA510UnitVALU>], (instregex "^[SU]ABA_ZZZ_[BHSD]")>; // Arithmetic, absolute diff accum long -def : InstRW<[CortexA510MCWrite<8, 2, CortexA510UnitVALU>], (instregex "^[SU]ABAL[TB]_ZZZ_[HSD]")>; +def : InstRW<[CortexA510MCWrite<6, 2, CortexA510UnitVALU>], (instregex "^[SU]ABAL[TB]_ZZZ_[HSD]")>; // Arithmetic, absolute diff long def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^[SU]ABDL[TB]_ZZZ_[HSD]")>; @@ -651,20 +652,22 @@ def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], "^(ADD|SUB|SUBR)_ZI_[BHSD]", "^ADR_[SU]XTW_ZZZ_D_[0123]", "^ADR_LSL_ZZZ_[SD]_[0123]", - "^[SU](ADD|SUB)[LW][BT]_ZZZ_[HSD]", + "^[SU]H(ADD|SUB|SUBR)_ZPmZ_[BHSD]")>; +def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], + (instregex "^[SU](ADD|SUB)[LW][BT]_ZZZ_[HSD]", "^SADDLBT_ZZZ_[HSD]", - "^[SU]H(ADD|SUB|SUBR)_ZPmZ_[BHSD]", "^SSUBL(BT|TB)_ZZZ_[HSD]")>; // Arithmetic, complex def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], - (instregex "^R?(ADD|SUB)HN[BT]_ZZZ_[BHS]", - "^SQ(ABS|NEG)_ZPmZ_[BHSD]", + (instregex "^SQ(ABS|NEG)_ZPmZ_[BHSD]", "^SQ(ADD|SUB|SUBR)_ZPmZ_?[BHSD]", "^[SU]Q(ADD|SUB)_ZZZ_[BHSD]", "^[SU]Q(ADD|SUB)_ZI_[BHSD]", "^(SRH|SUQ|UQ|USQ|URH)ADD_ZPmZ_[BHSD]", "^(UQSUB|UQSUBR)_ZPmZ_[BHSD]")>; +def : InstRW<[CortexA510Write<8, CortexA510UnitVALU>], + (instregex "^R?(ADD|SUB)HN[BT]_ZZZ_[BHS]")>; // Arithmetic, large integer def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^(AD|SB)CL[BT]_ZZZ_[SD]")>; @@ -735,14 +738,14 @@ def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^(BSL|BSL1N|B // Count/reverse bits def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^(CLS|CLZ|RBIT)_ZPmZ_[BHSD]")>; -def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^CNT_ZPmZ_[BH]")>; +def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^CNT_ZPmZ_[BH]")>; def : InstRW<[CortexA510Write<8, CortexA510UnitVALU>], (instregex "^CNT_ZPmZ_S")>; def : InstRW<[CortexA510Write<12, CortexA510UnitVALU>], (instregex "^CNT_ZPmZ_D")>; // Broadcast logical bitmask immediate to vector def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instrs DUPM_ZI)>; // Compare and set flags -def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], +def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^CMP(EQ|GE|GT|HI|HS|LE|LO|LS|LT|NE)_PPzZ[IZ]_[BHSD]", "^CMP(EQ|GE|GT|HI|HS|LE|LO|LS|LT|NE)_WIDE_PPzZZ_[BHS]")>; @@ -939,12 +942,14 @@ def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instregex "^SQRDMULH_ZZZ // Multiply/multiply long, (8x8) polynomial def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^PMUL_ZZZ_B")>; -def : InstRW<[CortexA510Write<6, CortexA510UnitVMC>], (instregex "^PMULL[BT]_ZZZ_[HDQ]")>; +def : InstRW<[CortexA510Write<9, CortexA510UnitVMC>], (instregex "^PMULL[BT]_ZZZ_[HDQ]")>; // Predicate counting vector +def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], + (instregex "^(DEC|INC)[HWD]_ZPiI")>; def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], - (instregex "^(DEC|INC|SQDEC|SQINC|UQDEC|UQINC)[HWD]_ZPiI")>; + (instregex "^(SQDEC|SQINC|UQDEC|UQINC)[HWD]_ZPiI")>; // Reciprocal estimate def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instregex "^URECPE_ZPmZ_S", "^URSQRTE_ZPmZ_S")>; @@ -965,7 +970,7 @@ def : InstRW<[CortexA510Write<4, CortexA510UnitVALU0>], (instregex "^[SU](ADD|MA def : InstRW<[CortexA510Write<4, CortexA510UnitVALU0>], (instregex "^(ANDV|EORV|ORV)_VPZ_[BHSD]")>; // Reverse, vector -def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^REV_ZZ_[BHSD]", +def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^REV_ZZ_[BHSD]", "^REVB_ZPmZ_[HSD]", "^REVH_ZPmZ_[SD]", "^REVW_ZPmZ_D")>; @@ -980,13 +985,13 @@ def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^TBL_ZZZZ?_[B def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^TBX_ZZZ_[BHSD]")>; // Transpose, vector form -def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^TRN[12]_ZZZ_[BHSDQ]")>; +def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^TRN[12]_ZZZ_[BHSDQ]")>; // Unpack and extend def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^[SU]UNPK(HI|LO)_ZZ_[HSD]")>; // Zip/unzip -def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^(UZP|ZIP)[12]_ZZZ_[BHSDQ]")>; +def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^(UZP|ZIP)[12]_ZZZ_[BHSDQ]")>; // SVE floating-point instructions // ----------------------------------------------------------------------------- @@ -1142,7 +1147,7 @@ def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instregex "^FTMAD_ZZI_[H // Floating point trigonometric, miscellaneous def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instregex "^FTSMUL_ZZZ_[HSD]")>; -def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^FTSSEL_ZZZ_[HSD]")>; +def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^FTSSEL_ZZZ_[HSD]")>; // SVE BFloat16 (BF16) instructions @@ -1251,12 +1256,12 @@ def : InstRW<[CortexA510MCWrite<7, 7, CortexA510UnitLdSt>], "^GLD(FF)?1D(_SCALED)?$")>; // Gather load, 32-bit scaled offset -def : InstRW<[CortexA510MCWrite<9, 9, CortexA510UnitLd>], +def : InstRW<[CortexA510MCWrite<7, 7, CortexA510UnitLd>], (instregex "^GLD(FF)?1S?[HW]_S_[SU]XTW_SCALED$", "^GLD(FF)?1W_[SU]XTW_SCALED")>; // Gather load, 32-bit unpacked unscaled offset -def : InstRW<[CortexA510MCWrite<9, 9, CortexA510UnitLd>], (instregex "^GLD(FF)?1S?[BH]_S_[SU]XTW$", +def : InstRW<[CortexA510MCWrite<7, 7, CortexA510UnitLd>], (instregex "^GLD(FF)?1S?[BH]_S_[SU]XTW$", "^GLD(FF)?1W_[SU]XTW$")>; def : InstRW<[CortexA510Write<0, CortexA510UnitVALU>], (instregex "^PRF(B|H|W|D).*")>; @@ -1377,12 +1382,12 @@ def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^AES[DE]_ZZZ_ "^AESI?MC_ZZ_B$")>; // Crypto SHA3 ops -def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^(BCAX|EOR3)_ZZZZ$", +def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^(BCAX|EOR3)_ZZZZ$", "^XAR_ZZZI_[BHSD]$")>; -def : InstRW<[CortexA510MC_RC0Write<8, CortexA510UnitVMC>], (instregex "^RAX1_ZZZ_D$")>; +def : InstRW<[CortexA510MC_RC0Write<9, CortexA510UnitVMC>], (instregex "^RAX1_ZZZ_D$")>; // Crypto SM4 ops -def : InstRW<[CortexA510MC_RC0Write<8, CortexA510UnitVMC>], (instregex "^SM4E(KEY)?_ZZZ_S$")>; +def : InstRW<[CortexA510MC_RC0Write<9, CortexA510UnitVMC>], (instregex "^SM4E(KEY)?_ZZZ_S$")>; } diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.ll b/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.ll index ceef0c49a45e..9a525151ca32 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.ll +++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.ll @@ -140,10 +140,10 @@ define <8 x i16> @combine_vec_udiv_nonuniform3(<8 x i16> %x) { ; GISEL-NEXT: umull2 v2.4s, v0.8h, v1.8h ; GISEL-NEXT: umull v1.4s, v0.4h, v1.4h ; GISEL-NEXT: uzp2 v1.8h, v1.8h, v2.8h +; GISEL-NEXT: ldr q2, [x8, :lo12:.LCPI3_0] ; GISEL-NEXT: sub v0.8h, v0.8h, v1.8h ; GISEL-NEXT: usra v1.8h, v0.8h, #1 -; GISEL-NEXT: ldr q0, [x8, :lo12:.LCPI3_0] -; GISEL-NEXT: neg v0.8h, v0.8h +; GISEL-NEXT: neg v0.8h, v2.8h ; GISEL-NEXT: ushl v0.8h, v1.8h, v0.8h ; GISEL-NEXT: ret %1 = udiv <8 x i16> %x, @@ -170,13 +170,13 @@ define <16 x i8> @combine_vec_udiv_nonuniform4(<16 x i8> %x) { ; GISEL-LABEL: combine_vec_udiv_nonuniform4: ; GISEL: // %bb.0: ; GISEL-NEXT: adrp x8, .LCPI4_2 +; GISEL-NEXT: adrp x9, .LCPI4_0 ; GISEL-NEXT: ldr q1, [x8, :lo12:.LCPI4_2] ; GISEL-NEXT: adrp x8, .LCPI4_1 +; GISEL-NEXT: ldr q4, [x9, :lo12:.LCPI4_0] ; GISEL-NEXT: ldr q3, [x8, :lo12:.LCPI4_1] -; GISEL-NEXT: adrp x8, .LCPI4_0 ; GISEL-NEXT: umull2 v2.8h, v0.16b, v1.16b ; GISEL-NEXT: umull v1.8h, v0.8b, v1.8b -; GISEL-NEXT: ldr q4, [x8, :lo12:.LCPI4_0] ; GISEL-NEXT: uzp2 v1.16b, v1.16b, v2.16b ; GISEL-NEXT: neg v2.16b, v3.16b ; GISEL-NEXT: shl v3.16b, v4.16b, #7 diff --git a/llvm/test/CodeGen/AArch64/aarch64-addv.ll b/llvm/test/CodeGen/AArch64/aarch64-addv.ll index b77d59134765..ee035ec1941d 100644 --- a/llvm/test/CodeGen/AArch64/aarch64-addv.ll +++ b/llvm/test/CodeGen/AArch64/aarch64-addv.ll @@ -101,12 +101,12 @@ define i32 @oversized_ADDV_256(ptr noalias nocapture readonly %arg1, ptr noalias ; GISEL-NEXT: ushll v2.8h, v2.8b, #0 ; GISEL-NEXT: usubl v3.4s, v1.4h, v2.4h ; GISEL-NEXT: usubl2 v1.4s, v1.8h, v2.8h -; GISEL-NEXT: neg v2.4s, v3.4s -; GISEL-NEXT: neg v4.4s, v1.4s -; GISEL-NEXT: cmgt v5.4s, v0.4s, v3.4s +; GISEL-NEXT: cmgt v2.4s, v0.4s, v3.4s ; GISEL-NEXT: cmgt v0.4s, v0.4s, v1.4s -; GISEL-NEXT: bif v2.16b, v3.16b, v5.16b -; GISEL-NEXT: bsl v0.16b, v4.16b, v1.16b +; GISEL-NEXT: neg v4.4s, v3.4s +; GISEL-NEXT: neg v5.4s, v1.4s +; GISEL-NEXT: bsl v2.16b, v4.16b, v3.16b +; GISEL-NEXT: bsl v0.16b, v5.16b, v1.16b ; GISEL-NEXT: add v0.4s, v2.4s, v0.4s ; GISEL-NEXT: addv s0, v0.4s ; GISEL-NEXT: fmov w0, s0 diff --git a/llvm/test/CodeGen/AArch64/aarch64-dup-ext-scalable.ll b/llvm/test/CodeGen/AArch64/aarch64-dup-ext-scalable.ll index fdeae9f326ad..36b81d8e495c 100644 --- a/llvm/test/CodeGen/AArch64/aarch64-dup-ext-scalable.ll +++ b/llvm/test/CodeGen/AArch64/aarch64-dup-ext-scalable.ll @@ -4,8 +4,8 @@ define @dupsext_v2i8_v2i16(i8 %src, %b) { ; CHECK-LABEL: dupsext_v2i8_v2i16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: sxtb w8, w0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret @@ -20,8 +20,8 @@ entry: define @dupsext_v4i8_v4i16(i8 %src, %b) { ; CHECK-LABEL: dupsext_v4i8_v4i16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sxtb w8, w0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: mul z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret @@ -36,8 +36,8 @@ entry: define @dupsext_v8i8_v8i16(i8 %src, %b) { ; CHECK-LABEL: dupsext_v8i8_v8i16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: sxtb w8, w0 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: mul z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret @@ -52,8 +52,8 @@ entry: define @dupsext_v2i8_v2i32(i8 %src, %b) { ; CHECK-LABEL: dupsext_v2i8_v2i32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: sxtb w8, w0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret @@ -68,8 +68,8 @@ entry: define @dupsext_v4i8_v4i32(i8 %src, %b) { ; CHECK-LABEL: dupsext_v4i8_v4i32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sxtb w8, w0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: mul z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret @@ -84,9 +84,9 @@ entry: define @dupsext_v2i8_v2i64(i8 %src, %b) { ; CHECK-LABEL: dupsext_v2i8_v2i64: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 ; CHECK-NEXT: sxtb x8, w0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret @@ -101,8 +101,8 @@ entry: define @dupsext_v2i16_v2i32(i16 %src, %b) { ; CHECK-LABEL: dupsext_v2i16_v2i32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: sxth w8, w0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret @@ -117,8 +117,8 @@ entry: define @dupsext_v4i16_v4i32(i16 %src, %b) { ; CHECK-LABEL: dupsext_v4i16_v4i32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sxth w8, w0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: mul z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret @@ -133,9 +133,9 @@ entry: define @dupsext_v2i16_v2i64(i16 %src, %b) { ; CHECK-LABEL: dupsext_v2i16_v2i64: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 ; CHECK-NEXT: sxth x8, w0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret @@ -150,9 +150,9 @@ entry: define @dupsext_v2i32_v2i64(i32 %src, %b) { ; CHECK-LABEL: dupsext_v2i32_v2i64: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 ; CHECK-NEXT: sxtw x8, w0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret @@ -167,8 +167,8 @@ entry: define @dupzext_v2i8_v2i16(i8 %src, %b) { ; CHECK-LABEL: dupzext_v2i8_v2i16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and w8, w0, #0xff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret @@ -183,8 +183,8 @@ entry: define @dupzext_v4i8_v4i16(i8 %src, %b) { ; CHECK-LABEL: dupzext_v4i8_v4i16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and w8, w0, #0xff +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: mul z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret @@ -199,8 +199,8 @@ entry: define @dupzext_v8i8_v8i16(i8 %src, %b) { ; CHECK-LABEL: dupzext_v8i8_v8i16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: and w8, w0, #0xff +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: mul z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret @@ -215,8 +215,8 @@ entry: define @dupzext_v2i8_v2i32(i8 %src, %b) { ; CHECK-LABEL: dupzext_v2i8_v2i32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and w8, w0, #0xff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret @@ -231,8 +231,8 @@ entry: define @dupzext_v4i8_v4i32(i8 %src, %b) { ; CHECK-LABEL: dupzext_v4i8_v4i32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and w8, w0, #0xff +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: mul z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret @@ -247,9 +247,9 @@ entry: define @dupzext_v2i8_v2i64(i8 %src, %b) { ; CHECK-LABEL: dupzext_v2i8_v2i64: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 ; CHECK-NEXT: and x8, x0, #0xff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret @@ -264,8 +264,8 @@ entry: define @dupzext_v2i16_v2i32(i16 %src, %b) { ; CHECK-LABEL: dupzext_v2i16_v2i32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and w8, w0, #0xffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret @@ -280,8 +280,8 @@ entry: define @dupzext_v4i16_v4i32(i16 %src, %b) { ; CHECK-LABEL: dupzext_v4i16_v4i32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and w8, w0, #0xffff +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: mul z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret @@ -296,9 +296,9 @@ entry: define @dupzext_v2i16_v2i64(i16 %src, %b) { ; CHECK-LABEL: dupzext_v2i16_v2i64: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 ; CHECK-NEXT: and x8, x0, #0xffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret @@ -313,8 +313,8 @@ entry: define @dupzext_v2i32_v2i64(i32 %src, %b) { ; CHECK-LABEL: dupzext_v2i32_v2i64: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/aarch64-smull.ll b/llvm/test/CodeGen/AArch64/aarch64-smull.ll index 61a4f64ac2bf..540471a05901 100644 --- a/llvm/test/CodeGen/AArch64/aarch64-smull.ll +++ b/llvm/test/CodeGen/AArch64/aarch64-smull.ll @@ -257,8 +257,8 @@ define <2 x i64> @smull_zext_v2i32_v2i64(ptr %A, ptr %B) nounwind { ; CHECK-SVE-LABEL: smull_zext_v2i32_v2i64: ; CHECK-SVE: // %bb.0: ; CHECK-SVE-NEXT: ldrh w8, [x0] -; CHECK-SVE-NEXT: ptrue p0.d, vl2 ; CHECK-SVE-NEXT: ldrh w9, [x0, #2] +; CHECK-SVE-NEXT: ptrue p0.d, vl2 ; CHECK-SVE-NEXT: ldr d0, [x1] ; CHECK-SVE-NEXT: fmov d1, x8 ; CHECK-SVE-NEXT: sshll v0.2d, v0.2s, #0 diff --git a/llvm/test/CodeGen/AArch64/active_lane_mask.ll b/llvm/test/CodeGen/AArch64/active_lane_mask.ll index a65c5d666779..43122c8c953f 100644 --- a/llvm/test/CodeGen/AArch64/active_lane_mask.ll +++ b/llvm/test/CodeGen/AArch64/active_lane_mask.ll @@ -131,9 +131,9 @@ define @lane_mask_nxv2i1_i8(i8 %index, i8 %TC) { ; CHECK-NEXT: index z0.d, #0, #1 ; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 ; CHECK-NEXT: and x8, x0, #0xff +; CHECK-NEXT: // kill: def $w1 killed $w1 def $x1 ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 -; CHECK-NEXT: // kill: def $w1 killed $w1 def $x1 ; CHECK-NEXT: and x8, x1, #0xff ; CHECK-NEXT: and z0.d, z0.d, #0xff ; CHECK-NEXT: add z0.d, z0.d, z1.d @@ -153,6 +153,7 @@ define @lane_mask_nxv32i1_i32(i32 %index, i32 %TC) { ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 +; CHECK-NEXT: str p7, [sp, #4, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: str p6, [sp, #5, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: str p5, [sp, #6, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill @@ -160,16 +161,16 @@ define @lane_mask_nxv32i1_i32(i32 %index, i32 %TC) { ; CHECK-NEXT: .cfi_offset w29, -16 ; CHECK-NEXT: index z0.s, #0, #1 ; CHECK-NEXT: mov z1.s, w0 -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z25.s, w1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.d, z0.d ; CHECK-NEXT: mov z3.d, z0.d ; CHECK-NEXT: uqadd z6.s, z0.s, z1.s ; CHECK-NEXT: incw z0.s, all, mul #4 ; CHECK-NEXT: incw z2.s ; CHECK-NEXT: incw z3.s, all, mul #2 -; CHECK-NEXT: cmphi p2.s, p0/z, z25.s, z6.s ; CHECK-NEXT: uqadd z0.s, z0.s, z1.s +; CHECK-NEXT: cmphi p2.s, p0/z, z25.s, z6.s ; CHECK-NEXT: mov z4.d, z2.d ; CHECK-NEXT: uqadd z5.s, z2.s, z1.s ; CHECK-NEXT: uqadd z7.s, z3.s, z1.s @@ -177,25 +178,26 @@ define @lane_mask_nxv32i1_i32(i32 %index, i32 %TC) { ; CHECK-NEXT: incw z3.s, all, mul #4 ; CHECK-NEXT: cmphi p5.s, p0/z, z25.s, z0.s ; CHECK-NEXT: incw z4.s, all, mul #2 -; CHECK-NEXT: cmphi p1.s, p0/z, z25.s, z5.s -; CHECK-NEXT: cmphi p3.s, p0/z, z25.s, z7.s ; CHECK-NEXT: uqadd z2.s, z2.s, z1.s ; CHECK-NEXT: uqadd z3.s, z3.s, z1.s +; CHECK-NEXT: cmphi p1.s, p0/z, z25.s, z5.s +; CHECK-NEXT: cmphi p3.s, p0/z, z25.s, z7.s ; CHECK-NEXT: uqadd z24.s, z4.s, z1.s ; CHECK-NEXT: incw z4.s, all, mul #4 -; CHECK-NEXT: uzp1 p1.h, p2.h, p1.h ; CHECK-NEXT: cmphi p6.s, p0/z, z25.s, z2.s -; CHECK-NEXT: cmphi p2.s, p0/z, z25.s, z3.s +; CHECK-NEXT: cmphi p7.s, p0/z, z25.s, z3.s +; CHECK-NEXT: uzp1 p1.h, p2.h, p1.h ; CHECK-NEXT: uqadd z1.s, z4.s, z1.s ; CHECK-NEXT: cmphi p4.s, p0/z, z25.s, z24.s -; CHECK-NEXT: uzp1 p3.h, p3.h, p4.h ; CHECK-NEXT: cmphi p0.s, p0/z, z25.s, z1.s -; CHECK-NEXT: uzp1 p4.h, p5.h, p6.h +; CHECK-NEXT: uzp1 p2.h, p3.h, p4.h +; CHECK-NEXT: uzp1 p3.h, p5.h, p6.h ; CHECK-NEXT: ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: uzp1 p2.h, p2.h, p0.h -; CHECK-NEXT: uzp1 p0.b, p1.b, p3.b -; CHECK-NEXT: uzp1 p1.b, p4.b, p2.b +; CHECK-NEXT: uzp1 p4.h, p7.h, p0.h +; CHECK-NEXT: ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload +; CHECK-NEXT: uzp1 p0.b, p1.b, p2.b +; CHECK-NEXT: uzp1 p1.b, p3.b, p4.b ; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: addvl sp, sp, #1 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload @@ -208,96 +210,97 @@ define @lane_mask_nxv32i1_i64(i64 %index, i64 %TC) { ; CHECK-LABEL: lane_mask_nxv32i1_i64: ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill -; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p10, [sp, #1, mul vl] // 2-byte Folded Spill +; CHECK-NEXT: addvl sp, sp, #-2 ; CHECK-NEXT: str p9, [sp, #2, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: str p8, [sp, #3, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: str p7, [sp, #4, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: str p6, [sp, #5, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: str p5, [sp, #6, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG +; CHECK-NEXT: str z8, [sp, #1, mul vl] // 16-byte Folded Spill +; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: index z1.d, #0, #1 +; CHECK-NEXT: .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 16 - 8 * VG +; CHECK-NEXT: index z5.d, #0, #1 ; CHECK-NEXT: mov z0.d, x0 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z3.d, x1 -; CHECK-NEXT: mov z2.d, z1.d -; CHECK-NEXT: mov z4.d, z1.d -; CHECK-NEXT: mov z6.d, z1.d -; CHECK-NEXT: uqadd z25.d, z1.d, z0.d -; CHECK-NEXT: incd z1.d, all, mul #8 +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: mov z2.d, z5.d +; CHECK-NEXT: mov z1.d, z5.d +; CHECK-NEXT: mov z4.d, z5.d +; CHECK-NEXT: uqadd z25.d, z5.d, z0.d +; CHECK-NEXT: incd z5.d, all, mul #8 ; CHECK-NEXT: incd z2.d -; CHECK-NEXT: incd z4.d, all, mul #2 -; CHECK-NEXT: incd z6.d, all, mul #4 -; CHECK-NEXT: cmphi p1.d, p0/z, z3.d, z25.d -; CHECK-NEXT: uqadd z1.d, z1.d, z0.d -; CHECK-NEXT: mov z5.d, z2.d -; CHECK-NEXT: uqadd z26.d, z2.d, z0.d +; CHECK-NEXT: incd z1.d, all, mul #2 +; CHECK-NEXT: incd z4.d, all, mul #4 +; CHECK-NEXT: uqadd z5.d, z5.d, z0.d +; CHECK-NEXT: cmphi p3.d, p0/z, z3.d, z25.d +; CHECK-NEXT: mov z6.d, z2.d ; CHECK-NEXT: mov z7.d, z2.d -; CHECK-NEXT: mov z24.d, z4.d -; CHECK-NEXT: uqadd z27.d, z4.d, z0.d -; CHECK-NEXT: uqadd z28.d, z6.d, z0.d +; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: uqadd z26.d, z2.d, z0.d +; CHECK-NEXT: uqadd z27.d, z1.d, z0.d +; CHECK-NEXT: uqadd z28.d, z4.d, z0.d ; CHECK-NEXT: incd z2.d, all, mul #8 +; CHECK-NEXT: incd z1.d, all, mul #8 ; CHECK-NEXT: incd z4.d, all, mul #8 -; CHECK-NEXT: incd z6.d, all, mul #8 -; CHECK-NEXT: incd z5.d, all, mul #2 +; CHECK-NEXT: incd z6.d, all, mul #2 ; CHECK-NEXT: incd z7.d, all, mul #4 -; CHECK-NEXT: cmphi p2.d, p0/z, z3.d, z26.d ; CHECK-NEXT: incd z24.d, all, mul #4 -; CHECK-NEXT: cmphi p3.d, p0/z, z3.d, z27.d -; CHECK-NEXT: cmphi p5.d, p0/z, z3.d, z28.d +; CHECK-NEXT: cmphi p4.d, p0/z, z3.d, z26.d +; CHECK-NEXT: cmphi p2.d, p0/z, z3.d, z27.d +; CHECK-NEXT: cmphi p1.d, p0/z, z3.d, z28.d +; CHECK-NEXT: mov z31.d, z6.d +; CHECK-NEXT: uqadd z29.d, z6.d, z0.d +; CHECK-NEXT: uqadd z30.d, z7.d, z0.d +; CHECK-NEXT: uqadd z8.d, z24.d, z0.d +; CHECK-NEXT: incd z6.d, all, mul #8 +; CHECK-NEXT: incd z7.d, all, mul #8 +; CHECK-NEXT: incd z24.d, all, mul #8 ; CHECK-NEXT: uqadd z2.d, z2.d, z0.d +; CHECK-NEXT: uqadd z1.d, z1.d, z0.d +; CHECK-NEXT: incd z31.d, all, mul #4 ; CHECK-NEXT: uqadd z4.d, z4.d, z0.d +; CHECK-NEXT: uzp1 p3.s, p3.s, p4.s +; CHECK-NEXT: cmphi p5.d, p0/z, z3.d, z29.d +; CHECK-NEXT: cmphi p7.d, p0/z, z3.d, z30.d ; CHECK-NEXT: uqadd z6.d, z6.d, z0.d -; CHECK-NEXT: mov z26.d, z5.d -; CHECK-NEXT: uqadd z25.d, z5.d, z0.d -; CHECK-NEXT: uqadd z27.d, z7.d, z0.d -; CHECK-NEXT: incd z5.d, all, mul #8 -; CHECK-NEXT: incd z7.d, all, mul #8 -; CHECK-NEXT: uzp1 p1.s, p1.s, p2.s -; CHECK-NEXT: incd z26.d, all, mul #4 -; CHECK-NEXT: cmphi p8.d, p0/z, z3.d, z2.d -; CHECK-NEXT: cmphi p4.d, p0/z, z3.d, z25.d -; CHECK-NEXT: uqadd z25.d, z24.d, z0.d -; CHECK-NEXT: incd z24.d, all, mul #8 -; CHECK-NEXT: uqadd z5.d, z5.d, z0.d +; CHECK-NEXT: cmphi p6.d, p0/z, z3.d, z8.d +; CHECK-NEXT: ldr z8, [sp, #1, mul vl] // 16-byte Folded Reload ; CHECK-NEXT: uqadd z7.d, z7.d, z0.d -; CHECK-NEXT: cmphi p6.d, p0/z, z3.d, z27.d -; CHECK-NEXT: uqadd z28.d, z26.d, z0.d -; CHECK-NEXT: incd z26.d, all, mul #8 -; CHECK-NEXT: uzp1 p3.s, p3.s, p4.s +; CHECK-NEXT: uqadd z25.d, z31.d, z0.d +; CHECK-NEXT: incd z31.d, all, mul #8 ; CHECK-NEXT: uqadd z24.d, z24.d, z0.d -; CHECK-NEXT: cmphi p7.d, p0/z, z3.d, z25.d -; CHECK-NEXT: cmphi p4.d, p0/z, z3.d, z1.d -; CHECK-NEXT: uzp1 p5.s, p5.s, p6.s -; CHECK-NEXT: cmphi p6.d, p0/z, z3.d, z4.d -; CHECK-NEXT: cmphi p9.d, p0/z, z3.d, z5.d -; CHECK-NEXT: cmphi p10.d, p0/z, z3.d, z7.d -; CHECK-NEXT: uqadd z0.d, z26.d, z0.d -; CHECK-NEXT: cmphi p2.d, p0/z, z3.d, z28.d -; CHECK-NEXT: uzp1 p4.s, p4.s, p8.s -; CHECK-NEXT: cmphi p8.d, p0/z, z3.d, z24.d -; CHECK-NEXT: uzp1 p6.s, p6.s, p9.s +; CHECK-NEXT: cmphi p4.d, p0/z, z3.d, z5.d +; CHECK-NEXT: uzp1 p2.s, p2.s, p5.s +; CHECK-NEXT: cmphi p5.d, p0/z, z3.d, z2.d +; CHECK-NEXT: cmphi p9.d, p0/z, z3.d, z6.d +; CHECK-NEXT: uqadd z0.d, z31.d, z0.d +; CHECK-NEXT: uzp1 p1.s, p1.s, p7.s +; CHECK-NEXT: cmphi p7.d, p0/z, z3.d, z1.d +; CHECK-NEXT: cmphi p8.d, p0/z, z3.d, z25.d +; CHECK-NEXT: uzp1 p2.h, p3.h, p2.h +; CHECK-NEXT: cmphi p3.d, p0/z, z3.d, z7.d +; CHECK-NEXT: uzp1 p4.s, p4.s, p5.s +; CHECK-NEXT: uzp1 p5.s, p7.s, p9.s ; CHECK-NEXT: ldr p9, [sp, #2, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: uzp1 p1.h, p1.h, p3.h -; CHECK-NEXT: uzp1 p2.s, p7.s, p2.s -; CHECK-NEXT: cmphi p7.d, p0/z, z3.d, z6.d +; CHECK-NEXT: uzp1 p6.s, p6.s, p8.s +; CHECK-NEXT: cmphi p8.d, p0/z, z3.d, z4.d +; CHECK-NEXT: ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload +; CHECK-NEXT: uzp1 p4.h, p4.h, p5.h +; CHECK-NEXT: ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload +; CHECK-NEXT: uzp1 p1.h, p1.h, p6.h +; CHECK-NEXT: cmphi p6.d, p0/z, z3.d, z24.d ; CHECK-NEXT: cmphi p0.d, p0/z, z3.d, z0.d -; CHECK-NEXT: uzp1 p7.s, p7.s, p10.s -; CHECK-NEXT: ldr p10, [sp, #1, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: uzp1 p0.s, p8.s, p0.s +; CHECK-NEXT: uzp1 p3.s, p8.s, p3.s ; CHECK-NEXT: ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: uzp1 p3.h, p4.h, p6.h +; CHECK-NEXT: uzp1 p0.s, p6.s, p0.s ; CHECK-NEXT: ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: uzp1 p2.h, p5.h, p2.h -; CHECK-NEXT: ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: uzp1 p4.h, p7.h, p0.h -; CHECK-NEXT: ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: uzp1 p0.b, p1.b, p2.b -; CHECK-NEXT: uzp1 p1.b, p3.b, p4.b +; CHECK-NEXT: uzp1 p3.h, p3.h, p0.h +; CHECK-NEXT: uzp1 p0.b, p2.b, p1.b +; CHECK-NEXT: uzp1 p1.b, p4.b, p3.b ; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: addvl sp, sp, #1 +; CHECK-NEXT: addvl sp, sp, #2 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload ; CHECK-NEXT: ret %active.lane.mask = call @llvm.get.active.lane.mask.nxv32i1.i64(i64 %index, i64 %TC) @@ -459,12 +462,12 @@ define <4 x i1> @lane_mask_v4i1_i8(i8 %index, i8 %TC) { ; CHECK-NEXT: adrp x8, .LCPI26_0 ; CHECK-NEXT: movi d2, #0xff00ff00ff00ff ; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI26_0] +; CHECK-NEXT: dup v3.4h, w1 ; CHECK-NEXT: bic v0.4h, #255, lsl #8 +; CHECK-NEXT: bic v3.4h, #255, lsl #8 ; CHECK-NEXT: add v0.4h, v0.4h, v1.4h -; CHECK-NEXT: dup v1.4h, w1 ; CHECK-NEXT: umin v0.4h, v0.4h, v2.4h -; CHECK-NEXT: bic v1.4h, #255, lsl #8 -; CHECK-NEXT: cmhi v0.4h, v1.4h, v0.4h +; CHECK-NEXT: cmhi v0.4h, v3.4h, v0.4h ; CHECK-NEXT: ret %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i8(i8 %index, i8 %TC) ret <4 x i1> %active.lane.mask @@ -480,9 +483,9 @@ define <2 x i1> @lane_mask_v2i1_i8(i8 %index, i8 %TC) { ; CHECK-NEXT: dup v3.2s, w1 ; CHECK-NEXT: and v1.8b, v1.8b, v0.8b ; CHECK-NEXT: add v1.2s, v1.2s, v2.2s -; CHECK-NEXT: umin v1.2s, v1.2s, v0.2s -; CHECK-NEXT: and v0.8b, v3.8b, v0.8b -; CHECK-NEXT: cmhi v0.2s, v0.2s, v1.2s +; CHECK-NEXT: and v2.8b, v3.8b, v0.8b +; CHECK-NEXT: umin v0.2s, v1.2s, v0.2s +; CHECK-NEXT: cmhi v0.2s, v2.2s, v0.2s ; CHECK-NEXT: ret %active.lane.mask = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i8(i8 %index, i8 %TC) ret <2 x i1> %active.lane.mask diff --git a/llvm/test/CodeGen/AArch64/arm64-convert-v4f64.ll b/llvm/test/CodeGen/AArch64/arm64-convert-v4f64.ll index 3007e7ce771e..508f68d6f14d 100644 --- a/llvm/test/CodeGen/AArch64/arm64-convert-v4f64.ll +++ b/llvm/test/CodeGen/AArch64/arm64-convert-v4f64.ll @@ -54,19 +54,19 @@ define <4 x bfloat> @uitofp_v4i64_to_v4bf16(ptr %ptr) { ; CHECK-LABEL: uitofp_v4i64_to_v4bf16: ; CHECK: // %bb.0: ; CHECK-NEXT: ldp q0, q1, [x0] -; CHECK-NEXT: movi v2.4s, #1 +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: ucvtf v0.2d, v0.2d ; CHECK-NEXT: ucvtf v1.2d, v1.2d ; CHECK-NEXT: fcvtn v0.2s, v0.2d ; CHECK-NEXT: fcvtn2 v0.4s, v1.2d -; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: movi v1.4s, #1 ; CHECK-NEXT: ushr v3.4s, v0.4s, #16 -; CHECK-NEXT: add v1.4s, v0.4s, v1.4s -; CHECK-NEXT: and v2.16b, v3.16b, v2.16b -; CHECK-NEXT: add v1.4s, v2.4s, v1.4s -; CHECK-NEXT: fcmeq v2.4s, v0.4s, v0.4s +; CHECK-NEXT: add v2.4s, v0.4s, v2.4s +; CHECK-NEXT: and v1.16b, v3.16b, v1.16b +; CHECK-NEXT: fcmeq v3.4s, v0.4s, v0.4s ; CHECK-NEXT: orr v0.4s, #64, lsl #16 -; CHECK-NEXT: bit v0.16b, v1.16b, v2.16b +; CHECK-NEXT: add v1.4s, v1.4s, v2.4s +; CHECK-NEXT: bit v0.16b, v1.16b, v3.16b ; CHECK-NEXT: shrn v0.4h, v0.4s, #16 ; CHECK-NEXT: ret %tmp1 = load <4 x i64>, ptr %ptr diff --git a/llvm/test/CodeGen/AArch64/arm64-vabs.ll b/llvm/test/CodeGen/AArch64/arm64-vabs.ll index fe4da2e7cf36..5b45ba2552ce 100644 --- a/llvm/test/CodeGen/AArch64/arm64-vabs.ll +++ b/llvm/test/CodeGen/AArch64/arm64-vabs.ll @@ -257,12 +257,12 @@ define i16 @uabd16b_rdx(ptr %a, ptr %b) { ; CHECK-GI-NEXT: movi.2d v0, #0000000000000000 ; CHECK-GI-NEXT: usubl.8h v3, v1, v2 ; CHECK-GI-NEXT: usubl2.8h v1, v1, v2 -; CHECK-GI-NEXT: neg.8h v2, v3 -; CHECK-GI-NEXT: neg.8h v4, v1 -; CHECK-GI-NEXT: cmgt.8h v5, v0, v3 +; CHECK-GI-NEXT: cmgt.8h v2, v0, v3 ; CHECK-GI-NEXT: cmgt.8h v0, v0, v1 -; CHECK-GI-NEXT: bif.16b v2, v3, v5 -; CHECK-GI-NEXT: bsl.16b v0, v4, v1 +; CHECK-GI-NEXT: neg.8h v4, v3 +; CHECK-GI-NEXT: neg.8h v5, v1 +; CHECK-GI-NEXT: bsl.16b v2, v4, v3 +; CHECK-GI-NEXT: bsl.16b v0, v5, v1 ; CHECK-GI-NEXT: add.8h v0, v2, v0 ; CHECK-GI-NEXT: addv.8h h0, v0 ; CHECK-GI-NEXT: fmov w0, s0 @@ -299,18 +299,18 @@ define i32 @uabd16b_rdx_i32(<16 x i8> %a, <16 x i8> %b) { ; CHECK-GI-NEXT: usubl2.4s v3, v3, v4 ; CHECK-GI-NEXT: usubl.4s v4, v0, v1 ; CHECK-GI-NEXT: usubl2.4s v0, v0, v1 -; CHECK-GI-NEXT: neg.4s v6, v5 -; CHECK-GI-NEXT: neg.4s v7, v3 ; CHECK-GI-NEXT: cmgt.4s v1, v2, v5 -; CHECK-GI-NEXT: neg.4s v16, v4 -; CHECK-GI-NEXT: neg.4s v17, v0 -; CHECK-GI-NEXT: cmgt.4s v18, v2, v3 -; CHECK-GI-NEXT: cmgt.4s v19, v2, v4 +; CHECK-GI-NEXT: cmgt.4s v6, v2, v3 +; CHECK-GI-NEXT: neg.4s v16, v5 +; CHECK-GI-NEXT: cmgt.4s v7, v2, v4 ; CHECK-GI-NEXT: cmgt.4s v2, v2, v0 -; CHECK-GI-NEXT: bsl.16b v1, v6, v5 -; CHECK-GI-NEXT: bit.16b v3, v7, v18 -; CHECK-GI-NEXT: bit.16b v4, v16, v19 -; CHECK-GI-NEXT: bit.16b v0, v17, v2 +; CHECK-GI-NEXT: neg.4s v17, v3 +; CHECK-GI-NEXT: neg.4s v18, v4 +; CHECK-GI-NEXT: neg.4s v19, v0 +; CHECK-GI-NEXT: bsl.16b v1, v16, v5 +; CHECK-GI-NEXT: bit.16b v3, v17, v6 +; CHECK-GI-NEXT: bit.16b v4, v18, v7 +; CHECK-GI-NEXT: bit.16b v0, v19, v2 ; CHECK-GI-NEXT: add.4s v1, v1, v3 ; CHECK-GI-NEXT: add.4s v0, v4, v0 ; CHECK-GI-NEXT: add.4s v0, v1, v0 @@ -347,18 +347,18 @@ define i32 @sabd16b_rdx_i32(<16 x i8> %a, <16 x i8> %b) { ; CHECK-GI-NEXT: ssubl2.4s v3, v3, v4 ; CHECK-GI-NEXT: ssubl.4s v4, v0, v1 ; CHECK-GI-NEXT: ssubl2.4s v0, v0, v1 -; CHECK-GI-NEXT: neg.4s v6, v5 -; CHECK-GI-NEXT: neg.4s v7, v3 ; CHECK-GI-NEXT: cmgt.4s v1, v2, v5 -; CHECK-GI-NEXT: neg.4s v16, v4 -; CHECK-GI-NEXT: neg.4s v17, v0 -; CHECK-GI-NEXT: cmgt.4s v18, v2, v3 -; CHECK-GI-NEXT: cmgt.4s v19, v2, v4 +; CHECK-GI-NEXT: cmgt.4s v6, v2, v3 +; CHECK-GI-NEXT: neg.4s v16, v5 +; CHECK-GI-NEXT: cmgt.4s v7, v2, v4 ; CHECK-GI-NEXT: cmgt.4s v2, v2, v0 -; CHECK-GI-NEXT: bsl.16b v1, v6, v5 -; CHECK-GI-NEXT: bit.16b v3, v7, v18 -; CHECK-GI-NEXT: bit.16b v4, v16, v19 -; CHECK-GI-NEXT: bit.16b v0, v17, v2 +; CHECK-GI-NEXT: neg.4s v17, v3 +; CHECK-GI-NEXT: neg.4s v18, v4 +; CHECK-GI-NEXT: neg.4s v19, v0 +; CHECK-GI-NEXT: bsl.16b v1, v16, v5 +; CHECK-GI-NEXT: bit.16b v3, v17, v6 +; CHECK-GI-NEXT: bit.16b v4, v18, v7 +; CHECK-GI-NEXT: bit.16b v0, v19, v2 ; CHECK-GI-NEXT: add.4s v1, v1, v3 ; CHECK-GI-NEXT: add.4s v0, v4, v0 ; CHECK-GI-NEXT: add.4s v0, v1, v0 @@ -396,12 +396,12 @@ define i32 @uabd8h_rdx(ptr %a, ptr %b) { ; CHECK-GI-NEXT: movi.2d v0, #0000000000000000 ; CHECK-GI-NEXT: usubl.4s v3, v1, v2 ; CHECK-GI-NEXT: usubl2.4s v1, v1, v2 -; CHECK-GI-NEXT: neg.4s v2, v3 -; CHECK-GI-NEXT: neg.4s v4, v1 -; CHECK-GI-NEXT: cmgt.4s v5, v0, v3 +; CHECK-GI-NEXT: cmgt.4s v2, v0, v3 ; CHECK-GI-NEXT: cmgt.4s v0, v0, v1 -; CHECK-GI-NEXT: bif.16b v2, v3, v5 -; CHECK-GI-NEXT: bsl.16b v0, v4, v1 +; CHECK-GI-NEXT: neg.4s v4, v3 +; CHECK-GI-NEXT: neg.4s v5, v1 +; CHECK-GI-NEXT: bsl.16b v2, v4, v3 +; CHECK-GI-NEXT: bsl.16b v0, v5, v1 ; CHECK-GI-NEXT: add.4s v0, v2, v0 ; CHECK-GI-NEXT: addv.4s s0, v0 ; CHECK-GI-NEXT: fmov w0, s0 @@ -428,15 +428,15 @@ define i32 @sabd8h_rdx(<8 x i16> %a, <8 x i16> %b) { ; ; CHECK-GI-LABEL: sabd8h_rdx: ; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi.2d v2, #0000000000000000 ; CHECK-GI-NEXT: ssubl.4s v3, v0, v1 ; CHECK-GI-NEXT: ssubl2.4s v0, v0, v1 -; CHECK-GI-NEXT: movi.2d v2, #0000000000000000 -; CHECK-GI-NEXT: neg.4s v1, v3 -; CHECK-GI-NEXT: neg.4s v4, v0 -; CHECK-GI-NEXT: cmgt.4s v5, v2, v3 +; CHECK-GI-NEXT: neg.4s v4, v3 +; CHECK-GI-NEXT: neg.4s v5, v0 +; CHECK-GI-NEXT: cmgt.4s v1, v2, v3 ; CHECK-GI-NEXT: cmgt.4s v2, v2, v0 -; CHECK-GI-NEXT: bif.16b v1, v3, v5 -; CHECK-GI-NEXT: bit.16b v0, v4, v2 +; CHECK-GI-NEXT: bsl.16b v1, v4, v3 +; CHECK-GI-NEXT: bit.16b v0, v5, v2 ; CHECK-GI-NEXT: add.4s v0, v1, v0 ; CHECK-GI-NEXT: addv.4s s0, v0 ; CHECK-GI-NEXT: fmov w0, s0 @@ -461,10 +461,10 @@ define i32 @uabdl4s_rdx_i32(<4 x i16> %a, <4 x i16> %b) { ; ; CHECK-GI-LABEL: uabdl4s_rdx_i32: ; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi.2d v2, #0000000000000000 ; CHECK-GI-NEXT: usubl.4s v0, v0, v1 -; CHECK-GI-NEXT: movi.2d v1, #0000000000000000 +; CHECK-GI-NEXT: cmgt.4s v1, v2, v0 ; CHECK-GI-NEXT: neg.4s v2, v0 -; CHECK-GI-NEXT: cmgt.4s v1, v1, v0 ; CHECK-GI-NEXT: bit.16b v0, v2, v1 ; CHECK-GI-NEXT: addv.4s s0, v0 ; CHECK-GI-NEXT: fmov w0, s0 @@ -499,12 +499,12 @@ define i64 @uabd4s_rdx(ptr %a, ptr %b, i32 %h) { ; CHECK-GI-NEXT: movi.2d v0, #0000000000000000 ; CHECK-GI-NEXT: usubl.2d v3, v1, v2 ; CHECK-GI-NEXT: usubl2.2d v1, v1, v2 -; CHECK-GI-NEXT: neg.2d v2, v3 -; CHECK-GI-NEXT: neg.2d v4, v1 -; CHECK-GI-NEXT: cmgt.2d v5, v0, v3 +; CHECK-GI-NEXT: cmgt.2d v2, v0, v3 ; CHECK-GI-NEXT: cmgt.2d v0, v0, v1 -; CHECK-GI-NEXT: bif.16b v2, v3, v5 -; CHECK-GI-NEXT: bsl.16b v0, v4, v1 +; CHECK-GI-NEXT: neg.2d v4, v3 +; CHECK-GI-NEXT: neg.2d v5, v1 +; CHECK-GI-NEXT: bsl.16b v2, v4, v3 +; CHECK-GI-NEXT: bsl.16b v0, v5, v1 ; CHECK-GI-NEXT: add.2d v0, v2, v0 ; CHECK-GI-NEXT: addp.2d d0, v0 ; CHECK-GI-NEXT: fmov x0, d0 @@ -531,15 +531,15 @@ define i64 @sabd4s_rdx(<4 x i32> %a, <4 x i32> %b) { ; ; CHECK-GI-LABEL: sabd4s_rdx: ; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi.2d v2, #0000000000000000 ; CHECK-GI-NEXT: ssubl.2d v3, v0, v1 ; CHECK-GI-NEXT: ssubl2.2d v0, v0, v1 -; CHECK-GI-NEXT: movi.2d v2, #0000000000000000 -; CHECK-GI-NEXT: neg.2d v1, v3 -; CHECK-GI-NEXT: neg.2d v4, v0 -; CHECK-GI-NEXT: cmgt.2d v5, v2, v3 +; CHECK-GI-NEXT: neg.2d v4, v3 +; CHECK-GI-NEXT: neg.2d v5, v0 +; CHECK-GI-NEXT: cmgt.2d v1, v2, v3 ; CHECK-GI-NEXT: cmgt.2d v2, v2, v0 -; CHECK-GI-NEXT: bif.16b v1, v3, v5 -; CHECK-GI-NEXT: bit.16b v0, v4, v2 +; CHECK-GI-NEXT: bsl.16b v1, v4, v3 +; CHECK-GI-NEXT: bit.16b v0, v5, v2 ; CHECK-GI-NEXT: add.2d v0, v1, v0 ; CHECK-GI-NEXT: addp.2d d0, v0 ; CHECK-GI-NEXT: fmov x0, d0 @@ -564,10 +564,10 @@ define i64 @uabdl2d_rdx_i64(<2 x i32> %a, <2 x i32> %b) { ; ; CHECK-GI-LABEL: uabdl2d_rdx_i64: ; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi.2d v2, #0000000000000000 ; CHECK-GI-NEXT: usubl.2d v0, v0, v1 -; CHECK-GI-NEXT: movi.2d v1, #0000000000000000 +; CHECK-GI-NEXT: cmgt.2d v1, v2, v0 ; CHECK-GI-NEXT: neg.2d v2, v0 -; CHECK-GI-NEXT: cmgt.2d v1, v1, v0 ; CHECK-GI-NEXT: bit.16b v0, v2, v1 ; CHECK-GI-NEXT: addp.2d d0, v0 ; CHECK-GI-NEXT: fmov x0, d0 @@ -1796,10 +1796,10 @@ define <2 x i64> @uabd_i32(<2 x i32> %a, <2 x i32> %b) { ; ; CHECK-GI-LABEL: uabd_i32: ; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi.2d v2, #0000000000000000 ; CHECK-GI-NEXT: ssubl.2d v0, v0, v1 -; CHECK-GI-NEXT: movi.2d v1, #0000000000000000 +; CHECK-GI-NEXT: cmgt.2d v1, v2, v0 ; CHECK-GI-NEXT: neg.2d v2, v0 -; CHECK-GI-NEXT: cmgt.2d v1, v1, v0 ; CHECK-GI-NEXT: bit.16b v0, v2, v1 ; CHECK-GI-NEXT: ret %aext = sext <2 x i32> %a to <2 x i64> diff --git a/llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll b/llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll index cafee32ada68..d4cc154ac6af 100644 --- a/llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll +++ b/llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll @@ -205,15 +205,15 @@ define <2 x bfloat> @test_vcvt_bf16_f64(<2 x double> %v) nounwind readnone ssp { ; GENERIC-LABEL: test_vcvt_bf16_f64: ; GENERIC: // %bb.0: ; GENERIC-NEXT: fcvtxn v0.2s, v0.2d -; GENERIC-NEXT: movi.4s v1, #127, msl #8 -; GENERIC-NEXT: movi.4s v2, #1 +; GENERIC-NEXT: movi.4s v1, #1 +; GENERIC-NEXT: movi.4s v2, #127, msl #8 ; GENERIC-NEXT: ushr.4s v3, v0, #16 -; GENERIC-NEXT: add.4s v1, v0, v1 -; GENERIC-NEXT: and.16b v2, v3, v2 -; GENERIC-NEXT: add.4s v1, v2, v1 -; GENERIC-NEXT: fcmeq.4s v2, v0, v0 +; GENERIC-NEXT: add.4s v2, v0, v2 +; GENERIC-NEXT: and.16b v1, v3, v1 +; GENERIC-NEXT: fcmeq.4s v3, v0, v0 ; GENERIC-NEXT: orr.4s v0, #64, lsl #16 -; GENERIC-NEXT: bit.16b v0, v1, v2 +; GENERIC-NEXT: add.4s v1, v1, v2 +; GENERIC-NEXT: bit.16b v0, v1, v3 ; GENERIC-NEXT: shrn.4h v0, v0, #16 ; GENERIC-NEXT: ret ; @@ -238,15 +238,15 @@ define <2 x bfloat> @test_vcvt_bf16_f64(<2 x double> %v) nounwind readnone ssp { ; GISEL-LABEL: test_vcvt_bf16_f64: ; GISEL: // %bb.0: ; GISEL-NEXT: fcvtxn v0.2s, v0.2d -; GISEL-NEXT: movi.4s v1, #127, msl #8 -; GISEL-NEXT: movi.4s v2, #1 +; GISEL-NEXT: movi.4s v1, #1 +; GISEL-NEXT: movi.4s v2, #127, msl #8 ; GISEL-NEXT: ushr.4s v3, v0, #16 -; GISEL-NEXT: add.4s v1, v0, v1 -; GISEL-NEXT: and.16b v2, v3, v2 -; GISEL-NEXT: add.4s v1, v2, v1 -; GISEL-NEXT: fcmeq.4s v2, v0, v0 +; GISEL-NEXT: add.4s v2, v0, v2 +; GISEL-NEXT: and.16b v1, v3, v1 +; GISEL-NEXT: fcmeq.4s v3, v0, v0 ; GISEL-NEXT: orr.4s v0, #64, lsl #16 -; GISEL-NEXT: bit.16b v0, v1, v2 +; GISEL-NEXT: add.4s v1, v1, v2 +; GISEL-NEXT: bit.16b v0, v1, v3 ; GISEL-NEXT: shrn.4h v0, v0, #16 ; GISEL-NEXT: ret %vcvt1.i = fptrunc <2 x double> %v to <2 x bfloat> diff --git a/llvm/test/CodeGen/AArch64/arm64-vhadd.ll b/llvm/test/CodeGen/AArch64/arm64-vhadd.ll index dda610e5dd3c..e754f01daa2a 100644 --- a/llvm/test/CodeGen/AArch64/arm64-vhadd.ll +++ b/llvm/test/CodeGen/AArch64/arm64-vhadd.ll @@ -903,10 +903,10 @@ define <2 x i16> @hadd8x2_sext_lsr(<2 x i8> %src1, <2 x i8> %src2) { ; CHECK: // %bb.0: ; CHECK-NEXT: shl.2s v0, v0, #24 ; CHECK-NEXT: shl.2s v1, v1, #24 +; CHECK-NEXT: movi d2, #0x00ffff0000ffff ; CHECK-NEXT: sshr.2s v0, v0, #24 ; CHECK-NEXT: ssra.2s v0, v1, #24 -; CHECK-NEXT: movi d1, #0x00ffff0000ffff -; CHECK-NEXT: and.8b v0, v0, v1 +; CHECK-NEXT: and.8b v0, v0, v2 ; CHECK-NEXT: ushr.2s v0, v0, #1 ; CHECK-NEXT: ret %zextsrc1 = sext <2 x i8> %src1 to <2 x i16> @@ -968,10 +968,10 @@ define <4 x i16> @rhadd8_sext_lsr(<4 x i8> %src1, <4 x i8> %src2) { ; CHECK: // %bb.0: ; CHECK-NEXT: shl.4h v0, v0, #8 ; CHECK-NEXT: shl.4h v1, v1, #8 +; CHECK-NEXT: movi.4h v2, #1 ; CHECK-NEXT: sshr.4h v0, v0, #8 ; CHECK-NEXT: ssra.4h v0, v1, #8 -; CHECK-NEXT: movi.4h v1, #1 -; CHECK-NEXT: add.4h v0, v0, v1 +; CHECK-NEXT: add.4h v0, v0, v2 ; CHECK-NEXT: ushr.4h v0, v0, #1 ; CHECK-NEXT: ret %zextsrc1 = sext <4 x i8> %src1 to <4 x i16> diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-add-mull-scalable-contract.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-add-mull-scalable-contract.ll index ebf5ce20d4ec..86b1d5d195ff 100644 --- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-add-mull-scalable-contract.ll +++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-add-mull-scalable-contract.ll @@ -7,21 +7,22 @@ target triple = "aarch64-unknown-linux-gnu" define @mull_add( %a, %b, %c) { ; CHECK-LABEL: mull_add: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: uzp1 z6.d, z0.d, z1.d -; CHECK-NEXT: uzp2 z7.d, z2.d, z3.d -; CHECK-NEXT: uzp2 z0.d, z0.d, z1.d +; CHECK-NEXT: uzp2 z6.d, z0.d, z1.d +; CHECK-NEXT: uzp1 z0.d, z0.d, z1.d +; CHECK-NEXT: uzp2 z1.d, z2.d, z3.d +; CHECK-NEXT: uzp1 z2.d, z2.d, z3.d ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: uzp1 z1.d, z2.d, z3.d -; CHECK-NEXT: fmul z2.d, z6.d, z7.d -; CHECK-NEXT: fmul z3.d, z0.d, z7.d -; CHECK-NEXT: fmad z0.d, p0/m, z1.d, z2.d -; CHECK-NEXT: fnmsb z1.d, p0/m, z6.d, z3.d -; CHECK-NEXT: uzp2 z2.d, z4.d, z5.d -; CHECK-NEXT: uzp1 z3.d, z4.d, z5.d -; CHECK-NEXT: fadd z2.d, z0.d, z2.d +; CHECK-NEXT: fmul z7.d, z0.d, z1.d +; CHECK-NEXT: fmul z1.d, z6.d, z1.d +; CHECK-NEXT: movprfx z3, z7 +; CHECK-NEXT: fmla z3.d, p0/m, z6.d, z2.d +; CHECK-NEXT: fnmsb z0.d, p0/m, z2.d, z1.d +; CHECK-NEXT: uzp2 z1.d, z4.d, z5.d +; CHECK-NEXT: uzp1 z2.d, z4.d, z5.d +; CHECK-NEXT: fadd z2.d, z2.d, z0.d ; CHECK-NEXT: fadd z1.d, z3.d, z1.d -; CHECK-NEXT: zip1 z0.d, z1.d, z2.d -; CHECK-NEXT: zip2 z1.d, z1.d, z2.d +; CHECK-NEXT: zip1 z0.d, z2.d, z1.d +; CHECK-NEXT: zip2 z1.d, z2.d, z1.d ; CHECK-NEXT: ret entry: %strided.vec = tail call { , } @llvm.experimental.vector.deinterleave2.nxv4f64( %a) @@ -49,21 +50,21 @@ entry: define @mul_add_mull( %a, %b, %c, %d) { ; CHECK-LABEL: mul_add_mull: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z24.d, #0 // =0x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z25.d, z24.d ; CHECK-NEXT: mov z26.d, z24.d ; CHECK-NEXT: mov z27.d, z24.d +; CHECK-NEXT: fcmla z24.d, p0/m, z7.d, z5.d, #0 ; CHECK-NEXT: fcmla z25.d, p0/m, z2.d, z0.d, #0 ; CHECK-NEXT: fcmla z26.d, p0/m, z3.d, z1.d, #0 ; CHECK-NEXT: fcmla z27.d, p0/m, z6.d, z4.d, #0 -; CHECK-NEXT: fcmla z24.d, p0/m, z7.d, z5.d, #0 +; CHECK-NEXT: fcmla z24.d, p0/m, z7.d, z5.d, #90 ; CHECK-NEXT: fcmla z25.d, p0/m, z2.d, z0.d, #90 ; CHECK-NEXT: fcmla z26.d, p0/m, z3.d, z1.d, #90 ; CHECK-NEXT: fcmla z27.d, p0/m, z6.d, z4.d, #90 -; CHECK-NEXT: fcmla z24.d, p0/m, z7.d, z5.d, #90 -; CHECK-NEXT: fadd z0.d, z25.d, z27.d ; CHECK-NEXT: fadd z1.d, z26.d, z24.d +; CHECK-NEXT: fadd z0.d, z25.d, z27.d ; CHECK-NEXT: ret entry: %strided.vec = tail call { , } @llvm.experimental.vector.deinterleave2.nxv4f64( %a) @@ -100,21 +101,21 @@ entry: define @mul_sub_mull( %a, %b, %c, %d) { ; CHECK-LABEL: mul_sub_mull: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z24.d, #0 // =0x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z25.d, z24.d ; CHECK-NEXT: mov z26.d, z24.d ; CHECK-NEXT: mov z27.d, z24.d +; CHECK-NEXT: fcmla z24.d, p0/m, z7.d, z5.d, #0 ; CHECK-NEXT: fcmla z25.d, p0/m, z2.d, z0.d, #0 ; CHECK-NEXT: fcmla z26.d, p0/m, z3.d, z1.d, #0 ; CHECK-NEXT: fcmla z27.d, p0/m, z6.d, z4.d, #0 -; CHECK-NEXT: fcmla z24.d, p0/m, z7.d, z5.d, #0 +; CHECK-NEXT: fcmla z24.d, p0/m, z7.d, z5.d, #90 ; CHECK-NEXT: fcmla z25.d, p0/m, z2.d, z0.d, #90 ; CHECK-NEXT: fcmla z26.d, p0/m, z3.d, z1.d, #90 ; CHECK-NEXT: fcmla z27.d, p0/m, z6.d, z4.d, #90 -; CHECK-NEXT: fcmla z24.d, p0/m, z7.d, z5.d, #90 -; CHECK-NEXT: fsub z0.d, z25.d, z27.d ; CHECK-NEXT: fsub z1.d, z26.d, z24.d +; CHECK-NEXT: fsub z0.d, z25.d, z27.d ; CHECK-NEXT: ret entry: %strided.vec = tail call { , } @llvm.experimental.vector.deinterleave2.nxv4f64( %a) @@ -151,21 +152,21 @@ entry: define @mul_conj_mull( %a, %b, %c, %d) { ; CHECK-LABEL: mul_conj_mull: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z24.d, #0 // =0x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z25.d, z24.d ; CHECK-NEXT: mov z26.d, z24.d ; CHECK-NEXT: mov z27.d, z24.d +; CHECK-NEXT: fcmla z24.d, p0/m, z5.d, z7.d, #0 ; CHECK-NEXT: fcmla z25.d, p0/m, z2.d, z0.d, #0 ; CHECK-NEXT: fcmla z26.d, p0/m, z3.d, z1.d, #0 ; CHECK-NEXT: fcmla z27.d, p0/m, z4.d, z6.d, #0 -; CHECK-NEXT: fcmla z24.d, p0/m, z5.d, z7.d, #0 +; CHECK-NEXT: fcmla z24.d, p0/m, z5.d, z7.d, #270 ; CHECK-NEXT: fcmla z25.d, p0/m, z2.d, z0.d, #90 ; CHECK-NEXT: fcmla z26.d, p0/m, z3.d, z1.d, #90 ; CHECK-NEXT: fcmla z27.d, p0/m, z4.d, z6.d, #270 -; CHECK-NEXT: fcmla z24.d, p0/m, z5.d, z7.d, #270 -; CHECK-NEXT: fadd z0.d, z25.d, z27.d ; CHECK-NEXT: fadd z1.d, z26.d, z24.d +; CHECK-NEXT: fadd z0.d, z25.d, z27.d ; CHECK-NEXT: ret entry: %strided.vec = tail call { , } @llvm.experimental.vector.deinterleave2.nxv4f64( %a) @@ -206,8 +207,8 @@ define @mul_add_rot_mull( %a, @mul_add_rot_mull( %a, @mul_add_mull( %a, %b, %c, %d) { ; CHECK-LABEL: mul_add_mull: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z24.d, #0 // =0x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z25.d, z24.d -; CHECK-NEXT: fcmla z25.d, p0/m, z6.d, z4.d, #0 ; CHECK-NEXT: fcmla z24.d, p0/m, z7.d, z5.d, #0 -; CHECK-NEXT: fcmla z25.d, p0/m, z0.d, z2.d, #0 +; CHECK-NEXT: fcmla z25.d, p0/m, z6.d, z4.d, #0 ; CHECK-NEXT: fcmla z24.d, p0/m, z1.d, z3.d, #0 -; CHECK-NEXT: fcmla z25.d, p0/m, z6.d, z4.d, #90 +; CHECK-NEXT: fcmla z25.d, p0/m, z0.d, z2.d, #0 ; CHECK-NEXT: fcmla z24.d, p0/m, z7.d, z5.d, #90 -; CHECK-NEXT: fcmla z25.d, p0/m, z0.d, z2.d, #90 +; CHECK-NEXT: fcmla z25.d, p0/m, z6.d, z4.d, #90 ; CHECK-NEXT: fcmla z24.d, p0/m, z1.d, z3.d, #90 -; CHECK-NEXT: mov z0.d, z25.d +; CHECK-NEXT: fcmla z25.d, p0/m, z0.d, z2.d, #90 ; CHECK-NEXT: mov z1.d, z24.d +; CHECK-NEXT: mov z0.d, z25.d ; CHECK-NEXT: ret entry: %strided.vec = tail call { , } @llvm.experimental.vector.deinterleave2.nxv4f64( %a) @@ -90,19 +90,19 @@ entry: define @mul_sub_mull( %a, %b, %c, %d) { ; CHECK-LABEL: mul_sub_mull: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z24.d, #0 // =0x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z25.d, z24.d -; CHECK-NEXT: fcmla z25.d, p0/m, z6.d, z4.d, #270 ; CHECK-NEXT: fcmla z24.d, p0/m, z7.d, z5.d, #270 -; CHECK-NEXT: fcmla z25.d, p0/m, z0.d, z2.d, #0 +; CHECK-NEXT: fcmla z25.d, p0/m, z6.d, z4.d, #270 ; CHECK-NEXT: fcmla z24.d, p0/m, z1.d, z3.d, #0 -; CHECK-NEXT: fcmla z25.d, p0/m, z6.d, z4.d, #180 +; CHECK-NEXT: fcmla z25.d, p0/m, z0.d, z2.d, #0 ; CHECK-NEXT: fcmla z24.d, p0/m, z7.d, z5.d, #180 -; CHECK-NEXT: fcmla z25.d, p0/m, z0.d, z2.d, #90 +; CHECK-NEXT: fcmla z25.d, p0/m, z6.d, z4.d, #180 ; CHECK-NEXT: fcmla z24.d, p0/m, z1.d, z3.d, #90 -; CHECK-NEXT: mov z0.d, z25.d +; CHECK-NEXT: fcmla z25.d, p0/m, z0.d, z2.d, #90 ; CHECK-NEXT: mov z1.d, z24.d +; CHECK-NEXT: mov z0.d, z25.d ; CHECK-NEXT: ret entry: %strided.vec = tail call { , } @llvm.experimental.vector.deinterleave2.nxv4f64( %a) @@ -139,19 +139,19 @@ entry: define @mul_conj_mull( %a, %b, %c, %d) { ; CHECK-LABEL: mul_conj_mull: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z24.d, #0 // =0x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z25.d, z24.d -; CHECK-NEXT: fcmla z25.d, p0/m, z0.d, z2.d, #0 ; CHECK-NEXT: fcmla z24.d, p0/m, z1.d, z3.d, #0 -; CHECK-NEXT: fcmla z25.d, p0/m, z0.d, z2.d, #90 +; CHECK-NEXT: fcmla z25.d, p0/m, z0.d, z2.d, #0 ; CHECK-NEXT: fcmla z24.d, p0/m, z1.d, z3.d, #90 -; CHECK-NEXT: fcmla z25.d, p0/m, z4.d, z6.d, #0 +; CHECK-NEXT: fcmla z25.d, p0/m, z0.d, z2.d, #90 ; CHECK-NEXT: fcmla z24.d, p0/m, z5.d, z7.d, #0 -; CHECK-NEXT: fcmla z25.d, p0/m, z4.d, z6.d, #270 +; CHECK-NEXT: fcmla z25.d, p0/m, z4.d, z6.d, #0 ; CHECK-NEXT: fcmla z24.d, p0/m, z5.d, z7.d, #270 -; CHECK-NEXT: mov z0.d, z25.d +; CHECK-NEXT: fcmla z25.d, p0/m, z4.d, z6.d, #270 ; CHECK-NEXT: mov z1.d, z24.d +; CHECK-NEXT: mov z0.d, z25.d ; CHECK-NEXT: ret entry: %strided.vec = tail call { , } @llvm.experimental.vector.deinterleave2.nxv4f64( %a) @@ -188,24 +188,25 @@ entry: define @mul_add_rot_mull( %a, %b, %c, %d) { ; CHECK-LABEL: mul_add_rot_mull: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: uzp1 z24.d, z2.d, z3.d +; CHECK-NEXT: uzp2 z24.d, z2.d, z3.d ; CHECK-NEXT: uzp2 z25.d, z0.d, z1.d -; CHECK-NEXT: uzp2 z2.d, z2.d, z3.d -; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: uzp1 z2.d, z2.d, z3.d ; CHECK-NEXT: uzp1 z0.d, z0.d, z1.d +; CHECK-NEXT: uzp2 z1.d, z4.d, z5.d ; CHECK-NEXT: uzp1 z26.d, z6.d, z7.d -; CHECK-NEXT: fmul z1.d, z24.d, z25.d -; CHECK-NEXT: fmul z3.d, z2.d, z25.d -; CHECK-NEXT: uzp2 z25.d, z4.d, z5.d +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: uzp1 z4.d, z4.d, z5.d ; CHECK-NEXT: uzp2 z5.d, z6.d, z7.d -; CHECK-NEXT: fmla z3.d, p0/m, z26.d, z25.d -; CHECK-NEXT: fmla z1.d, p0/m, z2.d, z0.d -; CHECK-NEXT: movprfx z2, z3 -; CHECK-NEXT: fmla z2.d, p0/m, z5.d, z4.d -; CHECK-NEXT: fmla z1.d, p0/m, z26.d, z4.d -; CHECK-NEXT: fnmls z2.d, p0/m, z24.d, z0.d -; CHECK-NEXT: fmls z1.d, p0/m, z5.d, z25.d +; CHECK-NEXT: fmul z3.d, z2.d, z25.d +; CHECK-NEXT: fmul z25.d, z24.d, z25.d +; CHECK-NEXT: fmla z3.d, p0/m, z24.d, z0.d +; CHECK-NEXT: movprfx z24, z25 +; CHECK-NEXT: fmla z24.d, p0/m, z26.d, z1.d +; CHECK-NEXT: movprfx z6, z24 +; CHECK-NEXT: fmla z6.d, p0/m, z5.d, z4.d +; CHECK-NEXT: fmla z3.d, p0/m, z26.d, z4.d +; CHECK-NEXT: fnmsb z2.d, p0/m, z0.d, z6.d +; CHECK-NEXT: fmsb z1.d, p0/m, z5.d, z3.d ; CHECK-NEXT: zip1 z0.d, z2.d, z1.d ; CHECK-NEXT: zip2 z1.d, z2.d, z1.d ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-f16-mul-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-f16-mul-scalable.ll index 611cf44ea7ee..cb285c05b2e8 100644 --- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-f16-mul-scalable.ll +++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-f16-mul-scalable.ll @@ -16,9 +16,10 @@ define @complex_mul_v4f16( %a, @complex_mul_v8f16( %a, %b) { ; CHECK-LABEL: complex_mul_v8f16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, #0 // =0x0 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: fcmla z2.h, p0/m, z1.h, z0.h, #0 ; CHECK-NEXT: fcmla z2.h, p0/m, z1.h, z0.h, #90 ; CHECK-NEXT: mov z0.d, z2.d @@ -72,15 +73,15 @@ entry: define @complex_mul_v16f16( %a, %b) { ; CHECK-LABEL: complex_mul_v16f16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z4.h, #0 // =0x0 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z5.d, z4.d -; CHECK-NEXT: fcmla z5.h, p0/m, z2.h, z0.h, #0 ; CHECK-NEXT: fcmla z4.h, p0/m, z3.h, z1.h, #0 -; CHECK-NEXT: fcmla z5.h, p0/m, z2.h, z0.h, #90 +; CHECK-NEXT: fcmla z5.h, p0/m, z2.h, z0.h, #0 ; CHECK-NEXT: fcmla z4.h, p0/m, z3.h, z1.h, #90 -; CHECK-NEXT: mov z0.d, z5.d +; CHECK-NEXT: fcmla z5.h, p0/m, z2.h, z0.h, #90 ; CHECK-NEXT: mov z1.d, z4.d +; CHECK-NEXT: mov z0.d, z5.d ; CHECK-NEXT: ret entry: %a.deinterleaved = tail call { , } @llvm.experimental.vector.deinterleave2.nxv16f16( %a) @@ -103,23 +104,23 @@ entry: define @complex_mul_v32f16( %a, %b) { ; CHECK-LABEL: complex_mul_v32f16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z24.h, #0 // =0x0 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z25.d, z24.d ; CHECK-NEXT: mov z26.d, z24.d ; CHECK-NEXT: mov z27.d, z24.d +; CHECK-NEXT: fcmla z24.h, p0/m, z7.h, z3.h, #0 ; CHECK-NEXT: fcmla z25.h, p0/m, z4.h, z0.h, #0 ; CHECK-NEXT: fcmla z26.h, p0/m, z5.h, z1.h, #0 ; CHECK-NEXT: fcmla z27.h, p0/m, z6.h, z2.h, #0 -; CHECK-NEXT: fcmla z24.h, p0/m, z7.h, z3.h, #0 +; CHECK-NEXT: fcmla z24.h, p0/m, z7.h, z3.h, #90 ; CHECK-NEXT: fcmla z25.h, p0/m, z4.h, z0.h, #90 ; CHECK-NEXT: fcmla z26.h, p0/m, z5.h, z1.h, #90 ; CHECK-NEXT: fcmla z27.h, p0/m, z6.h, z2.h, #90 -; CHECK-NEXT: fcmla z24.h, p0/m, z7.h, z3.h, #90 +; CHECK-NEXT: mov z3.d, z24.d ; CHECK-NEXT: mov z0.d, z25.d ; CHECK-NEXT: mov z1.d, z26.d ; CHECK-NEXT: mov z2.d, z27.d -; CHECK-NEXT: mov z3.d, z24.d ; CHECK-NEXT: ret entry: %a.deinterleaved = tail call { , } @llvm.experimental.vector.deinterleave2.nxv32f16( %a) diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-f32-mul-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-f32-mul-scalable.ll index 0f5e9a2202dd..1e2afb78de1b 100644 --- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-f32-mul-scalable.ll +++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-f32-mul-scalable.ll @@ -7,8 +7,8 @@ target triple = "aarch64" define @complex_mul_v4f32( %a, %b) { ; CHECK-LABEL: complex_mul_v4f32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, #0 // =0x0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: fcmla z2.s, p0/m, z1.s, z0.s, #0 ; CHECK-NEXT: fcmla z2.s, p0/m, z1.s, z0.s, #90 ; CHECK-NEXT: mov z0.d, z2.d @@ -34,15 +34,15 @@ entry: define @complex_mul_v8f32( %a, %b) { ; CHECK-LABEL: complex_mul_v8f32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z4.s, #0 // =0x0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z5.d, z4.d -; CHECK-NEXT: fcmla z5.s, p0/m, z2.s, z0.s, #0 ; CHECK-NEXT: fcmla z4.s, p0/m, z3.s, z1.s, #0 -; CHECK-NEXT: fcmla z5.s, p0/m, z2.s, z0.s, #90 +; CHECK-NEXT: fcmla z5.s, p0/m, z2.s, z0.s, #0 ; CHECK-NEXT: fcmla z4.s, p0/m, z3.s, z1.s, #90 -; CHECK-NEXT: mov z0.d, z5.d +; CHECK-NEXT: fcmla z5.s, p0/m, z2.s, z0.s, #90 ; CHECK-NEXT: mov z1.d, z4.d +; CHECK-NEXT: mov z0.d, z5.d ; CHECK-NEXT: ret entry: %a.deinterleaved = tail call { , } @llvm.experimental.vector.deinterleave2.nxv8f32( %a) @@ -65,23 +65,23 @@ entry: define @complex_mul_v16f32( %a, %b) { ; CHECK-LABEL: complex_mul_v16f32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z24.s, #0 // =0x0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z25.d, z24.d ; CHECK-NEXT: mov z26.d, z24.d ; CHECK-NEXT: mov z27.d, z24.d +; CHECK-NEXT: fcmla z24.s, p0/m, z7.s, z3.s, #0 ; CHECK-NEXT: fcmla z25.s, p0/m, z4.s, z0.s, #0 ; CHECK-NEXT: fcmla z26.s, p0/m, z5.s, z1.s, #0 ; CHECK-NEXT: fcmla z27.s, p0/m, z6.s, z2.s, #0 -; CHECK-NEXT: fcmla z24.s, p0/m, z7.s, z3.s, #0 +; CHECK-NEXT: fcmla z24.s, p0/m, z7.s, z3.s, #90 ; CHECK-NEXT: fcmla z25.s, p0/m, z4.s, z0.s, #90 ; CHECK-NEXT: fcmla z26.s, p0/m, z5.s, z1.s, #90 ; CHECK-NEXT: fcmla z27.s, p0/m, z6.s, z2.s, #90 -; CHECK-NEXT: fcmla z24.s, p0/m, z7.s, z3.s, #90 +; CHECK-NEXT: mov z3.d, z24.d ; CHECK-NEXT: mov z0.d, z25.d ; CHECK-NEXT: mov z1.d, z26.d ; CHECK-NEXT: mov z2.d, z27.d -; CHECK-NEXT: mov z3.d, z24.d ; CHECK-NEXT: ret entry: %a.deinterleaved = tail call { , } @llvm.experimental.vector.deinterleave2.nxv16f32( %a) diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-f64-mul-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-f64-mul-scalable.ll index 1fe554bdc616..17a239a09a03 100644 --- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-f64-mul-scalable.ll +++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-f64-mul-scalable.ll @@ -7,8 +7,8 @@ target triple = "aarch64" define @complex_mul_v2f64( %a, %b) { ; CHECK-LABEL: complex_mul_v2f64: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z2.d, #0 // =0x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: fcmla z2.d, p0/m, z1.d, z0.d, #0 ; CHECK-NEXT: fcmla z2.d, p0/m, z1.d, z0.d, #90 ; CHECK-NEXT: mov z0.d, z2.d @@ -34,15 +34,15 @@ entry: define @complex_mul_v4f64( %a, %b) { ; CHECK-LABEL: complex_mul_v4f64: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z4.d, #0 // =0x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z5.d, z4.d -; CHECK-NEXT: fcmla z5.d, p0/m, z2.d, z0.d, #0 ; CHECK-NEXT: fcmla z4.d, p0/m, z3.d, z1.d, #0 -; CHECK-NEXT: fcmla z5.d, p0/m, z2.d, z0.d, #90 +; CHECK-NEXT: fcmla z5.d, p0/m, z2.d, z0.d, #0 ; CHECK-NEXT: fcmla z4.d, p0/m, z3.d, z1.d, #90 -; CHECK-NEXT: mov z0.d, z5.d +; CHECK-NEXT: fcmla z5.d, p0/m, z2.d, z0.d, #90 ; CHECK-NEXT: mov z1.d, z4.d +; CHECK-NEXT: mov z0.d, z5.d ; CHECK-NEXT: ret entry: %a.deinterleaved = tail call { , } @llvm.experimental.vector.deinterleave2.nxv4f64( %a) @@ -65,23 +65,23 @@ entry: define @complex_mul_v8f64( %a, %b) { ; CHECK-LABEL: complex_mul_v8f64: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z24.d, #0 // =0x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z25.d, z24.d ; CHECK-NEXT: mov z26.d, z24.d ; CHECK-NEXT: mov z27.d, z24.d +; CHECK-NEXT: fcmla z24.d, p0/m, z7.d, z3.d, #0 ; CHECK-NEXT: fcmla z25.d, p0/m, z4.d, z0.d, #0 ; CHECK-NEXT: fcmla z26.d, p0/m, z5.d, z1.d, #0 ; CHECK-NEXT: fcmla z27.d, p0/m, z6.d, z2.d, #0 -; CHECK-NEXT: fcmla z24.d, p0/m, z7.d, z3.d, #0 +; CHECK-NEXT: fcmla z24.d, p0/m, z7.d, z3.d, #90 ; CHECK-NEXT: fcmla z25.d, p0/m, z4.d, z0.d, #90 ; CHECK-NEXT: fcmla z26.d, p0/m, z5.d, z1.d, #90 ; CHECK-NEXT: fcmla z27.d, p0/m, z6.d, z2.d, #90 -; CHECK-NEXT: fcmla z24.d, p0/m, z7.d, z3.d, #90 +; CHECK-NEXT: mov z3.d, z24.d ; CHECK-NEXT: mov z0.d, z25.d ; CHECK-NEXT: mov z1.d, z26.d ; CHECK-NEXT: mov z2.d, z27.d -; CHECK-NEXT: mov z3.d, z24.d ; CHECK-NEXT: ret entry: %a.deinterleaved = tail call { , } @llvm.experimental.vector.deinterleave2.nxv8f64( %a) diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-i16-mul-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-i16-mul-scalable.ll index 1b8a21b66ade..07488b623b98 100644 --- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-i16-mul-scalable.ll +++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-i16-mul-scalable.ll @@ -16,8 +16,9 @@ define @complex_mul_v4i16( %a, This Inner Loop Header: Depth=1 -; CHECK-NEXT: zip2 p3.d, p1.d, p1.d +; CHECK-NEXT: zip2 p2.d, p1.d, p1.d ; CHECK-NEXT: add x13, x0, x8 ; CHECK-NEXT: add x14, x1, x8 -; CHECK-NEXT: zip1 p2.d, p1.d, p1.d +; CHECK-NEXT: zip1 p1.d, p1.d, p1.d ; CHECK-NEXT: mov z6.d, z1.d ; CHECK-NEXT: mov z7.d, z0.d -; CHECK-NEXT: whilelo p1.d, x12, x9 +; CHECK-NEXT: ld1d { z2.d }, p2/z, [x13, #1, mul vl] +; CHECK-NEXT: ld1d { z4.d }, p2/z, [x14, #1, mul vl] ; CHECK-NEXT: add x8, x8, x11 -; CHECK-NEXT: add x12, x12, x10 -; CHECK-NEXT: ld1d { z2.d }, p3/z, [x13, #1, mul vl] -; CHECK-NEXT: ld1d { z4.d }, p3/z, [x14, #1, mul vl] -; CHECK-NEXT: ld1d { z3.d }, p2/z, [x13] -; CHECK-NEXT: ld1d { z5.d }, p2/z, [x14] +; CHECK-NEXT: ld1d { z3.d }, p1/z, [x13] +; CHECK-NEXT: ld1d { z5.d }, p1/z, [x14] ; CHECK-NEXT: fcmla z7.d, p0/m, z4.d, z2.d, #0 ; CHECK-NEXT: fcmla z6.d, p0/m, z5.d, z3.d, #0 ; CHECK-NEXT: fcmla z7.d, p0/m, z4.d, z2.d, #90 ; CHECK-NEXT: fcmla z6.d, p0/m, z5.d, z3.d, #90 -; CHECK-NEXT: mov z0.d, p3/m, z7.d -; CHECK-NEXT: mov z1.d, p2/m, z6.d +; CHECK-NEXT: mov z0.d, p2/m, z7.d +; CHECK-NEXT: mov z1.d, p1/m, z6.d +; CHECK-NEXT: whilelo p1.d, x12, x9 +; CHECK-NEXT: add x12, x12, x10 ; CHECK-NEXT: b.mi .LBB0_1 ; CHECK-NEXT: // %bb.2: // %exit.block ; CHECK-NEXT: uzp1 z2.d, z1.d, z0.d @@ -114,10 +114,10 @@ define %"class.std::complex" @complex_mul_predicated_v2f64(ptr %a, ptr %b, ptr % ; CHECK-LABEL: complex_mul_predicated_v2f64: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: mov z1.d, #0 // =0x0 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: cntd x10 -; CHECK-NEXT: neg x11, x10 ; CHECK-NEXT: mov w12, #100 // =0x64 +; CHECK-NEXT: neg x11, x10 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, xzr ; CHECK-NEXT: mov x9, xzr ; CHECK-NEXT: and x11, x11, x12 @@ -133,20 +133,20 @@ define %"class.std::complex" @complex_mul_predicated_v2f64(ptr %a, ptr %b, ptr % ; CHECK-NEXT: mov z7.d, z0.d ; CHECK-NEXT: add x9, x9, x10 ; CHECK-NEXT: add x8, x8, x12 -; CHECK-NEXT: cmpne p1.d, p0/z, z2.d, #0 +; CHECK-NEXT: cmpne p2.d, p0/z, z2.d, #0 ; CHECK-NEXT: cmp x11, x9 -; CHECK-NEXT: zip2 p2.d, p1.d, p1.d -; CHECK-NEXT: zip1 p1.d, p1.d, p1.d -; CHECK-NEXT: ld1d { z2.d }, p2/z, [x13, #1, mul vl] -; CHECK-NEXT: ld1d { z4.d }, p2/z, [x14, #1, mul vl] -; CHECK-NEXT: ld1d { z3.d }, p1/z, [x13] -; CHECK-NEXT: ld1d { z5.d }, p1/z, [x14] +; CHECK-NEXT: zip2 p1.d, p2.d, p2.d +; CHECK-NEXT: zip1 p2.d, p2.d, p2.d +; CHECK-NEXT: ld1d { z2.d }, p1/z, [x13, #1, mul vl] +; CHECK-NEXT: ld1d { z4.d }, p1/z, [x14, #1, mul vl] +; CHECK-NEXT: ld1d { z3.d }, p2/z, [x13] +; CHECK-NEXT: ld1d { z5.d }, p2/z, [x14] ; CHECK-NEXT: fcmla z7.d, p0/m, z4.d, z2.d, #0 ; CHECK-NEXT: fcmla z6.d, p0/m, z5.d, z3.d, #0 ; CHECK-NEXT: fcmla z7.d, p0/m, z4.d, z2.d, #90 ; CHECK-NEXT: fcmla z6.d, p0/m, z5.d, z3.d, #90 -; CHECK-NEXT: mov z0.d, p2/m, z7.d -; CHECK-NEXT: mov z1.d, p1/m, z6.d +; CHECK-NEXT: mov z0.d, p1/m, z7.d +; CHECK-NEXT: mov z1.d, p2/m, z6.d ; CHECK-NEXT: b.ne .LBB1_1 ; CHECK-NEXT: // %bb.2: // %exit.block ; CHECK-NEXT: uzp1 z2.d, z1.d, z0.d @@ -217,8 +217,8 @@ exit.block: ; preds = %vector.body define %"class.std::complex" @complex_mul_predicated_x2_v2f64(ptr %a, ptr %b, ptr %cond) { ; CHECK-LABEL: complex_mul_predicated_x2_v2f64: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: mov w10, #100 // =0x64 ; CHECK-NEXT: mov z1.d, #0 // =0x0 +; CHECK-NEXT: mov w10, #100 // =0x64 ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: whilelo p1.d, xzr, x10 ; CHECK-NEXT: mov x8, xzr @@ -236,19 +236,19 @@ define %"class.std::complex" @complex_mul_predicated_x2_v2f64(ptr %a, ptr %b, pt ; CHECK-NEXT: mov z7.d, z0.d ; CHECK-NEXT: add x9, x9, x11 ; CHECK-NEXT: add x8, x8, x12 -; CHECK-NEXT: cmpne p1.d, p1/z, z2.d, #0 -; CHECK-NEXT: zip2 p3.d, p1.d, p1.d -; CHECK-NEXT: zip1 p2.d, p1.d, p1.d -; CHECK-NEXT: whilelo p1.d, x9, x10 -; CHECK-NEXT: ld1d { z2.d }, p3/z, [x13, #1, mul vl] -; CHECK-NEXT: ld1d { z4.d }, p3/z, [x14, #1, mul vl] +; CHECK-NEXT: cmpne p2.d, p1/z, z2.d, #0 +; CHECK-NEXT: zip2 p1.d, p2.d, p2.d +; CHECK-NEXT: zip1 p2.d, p2.d, p2.d +; CHECK-NEXT: ld1d { z2.d }, p1/z, [x13, #1, mul vl] +; CHECK-NEXT: ld1d { z4.d }, p1/z, [x14, #1, mul vl] ; CHECK-NEXT: ld1d { z3.d }, p2/z, [x13] ; CHECK-NEXT: ld1d { z5.d }, p2/z, [x14] ; CHECK-NEXT: fcmla z7.d, p0/m, z4.d, z2.d, #0 ; CHECK-NEXT: fcmla z6.d, p0/m, z5.d, z3.d, #0 ; CHECK-NEXT: fcmla z7.d, p0/m, z4.d, z2.d, #90 ; CHECK-NEXT: fcmla z6.d, p0/m, z5.d, z3.d, #90 -; CHECK-NEXT: mov z0.d, p3/m, z7.d +; CHECK-NEXT: mov z0.d, p1/m, z7.d +; CHECK-NEXT: whilelo p1.d, x9, x10 ; CHECK-NEXT: mov z1.d, p2/m, z6.d ; CHECK-NEXT: b.mi .LBB2_1 ; CHECK-NEXT: // %bb.2: // %exit.block diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll index 1696ac8709d4..664d99a3627b 100644 --- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll +++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll @@ -15,11 +15,11 @@ define %"class.std::complex" @complex_mul_v2f64(ptr %a, ptr %b) { ; CHECK-LABEL: complex_mul_v2f64: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: mov z1.d, #0 // =0x0 -; CHECK-NEXT: ptrue p1.b ; CHECK-NEXT: cntd x9 -; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: ptrue p1.b ; CHECK-NEXT: neg x9, x9 ; CHECK-NEXT: mov w10, #100 // =0x64 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, xzr ; CHECK-NEXT: and x10, x9, x10 ; CHECK-NEXT: rdvl x11, #2 @@ -101,18 +101,18 @@ exit.block: ; preds = %vector.body define %"class.std::complex" @complex_mul_nonzero_init_v2f64(ptr %a, ptr %b) { ; CHECK-LABEL: complex_mul_nonzero_init_v2f64: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d, vl1 ; CHECK-NEXT: fmov d0, #1.00000000 ; CHECK-NEXT: mov z1.d, #0 // =0x0 -; CHECK-NEXT: fmov d2, #2.00000000 ; CHECK-NEXT: cntd x9 -; CHECK-NEXT: mov w10, #100 // =0x64 -; CHECK-NEXT: ptrue p1.b +; CHECK-NEXT: fmov d2, #2.00000000 +; CHECK-NEXT: ptrue p0.d, vl1 ; CHECK-NEXT: neg x9, x9 +; CHECK-NEXT: ptrue p1.b +; CHECK-NEXT: mov w10, #100 // =0x64 ; CHECK-NEXT: mov x8, xzr +; CHECK-NEXT: sel z3.d, p0, z0.d, z1.d ; CHECK-NEXT: and x10, x9, x10 ; CHECK-NEXT: rdvl x11, #2 -; CHECK-NEXT: sel z3.d, p0, z0.d, z1.d ; CHECK-NEXT: mov z1.d, p0/m, z2.d ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: zip2 z0.d, z1.d, z3.d @@ -190,12 +190,12 @@ define %"class.std::complex" @complex_mul_v2f64_unrolled(ptr %a, ptr %b) { ; CHECK-LABEL: complex_mul_v2f64_unrolled: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: mov z1.d, #0 // =0x0 -; CHECK-NEXT: ptrue p1.b ; CHECK-NEXT: cntw x9 -; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: neg x9, x9 ; CHECK-NEXT: mov w10, #1000 // =0x3e8 +; CHECK-NEXT: neg x9, x9 ; CHECK-NEXT: rdvl x12, #2 +; CHECK-NEXT: ptrue p1.b +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, xzr ; CHECK-NEXT: and x10, x9, x10 ; CHECK-NEXT: zip2 z0.d, z1.d, z1.d @@ -324,10 +324,10 @@ define dso_local %"class.std::complex" @reduction_mix(ptr %a, ptr %b, ptr noalia ; CHECK-LABEL: reduction_mix: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: mov z2.d, #0 // =0x0 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: cntd x9 -; CHECK-NEXT: neg x10, x9 ; CHECK-NEXT: mov w11, #100 // =0x64 +; CHECK-NEXT: neg x10, x9 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, xzr ; CHECK-NEXT: and x10, x10, x11 ; CHECK-NEXT: rdvl x11, #2 @@ -349,8 +349,8 @@ define dso_local %"class.std::complex" @reduction_mix(ptr %a, ptr %b, ptr noalia ; CHECK-NEXT: uaddv d2, p0, z2.d ; CHECK-NEXT: uzp2 z3.d, z1.d, z0.d ; CHECK-NEXT: uzp1 z1.d, z1.d, z0.d -; CHECK-NEXT: fmov x8, d2 ; CHECK-NEXT: faddv d0, p0, z3.d +; CHECK-NEXT: fmov x8, d2 ; CHECK-NEXT: faddv d1, p0, z1.d ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 ; CHECK-NEXT: // kill: def $d1 killed $d1 killed $z1 diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-splat-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-splat-scalable.ll index 742a7099559f..17bf5ba6eb48 100644 --- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-splat-scalable.ll +++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-splat-scalable.ll @@ -8,8 +8,8 @@ target triple = "aarch64" define @complex_mul_const( %a, %b) { ; CHECK-LABEL: complex_mul_const: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z4.d, #0 // =0x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: fmov z7.d, #3.00000000 ; CHECK-NEXT: fmov z24.d, #11.00000000 ; CHECK-NEXT: mov z6.d, z4.d @@ -55,25 +55,25 @@ entry: define @complex_mul_non_const( %a, %b, [2 x double] %c) { ; CHECK-LABEL: complex_mul_non_const: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z6.d, #0 // =0x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $d5 killed $d5 def $z5 ; CHECK-NEXT: // kill: def $d4 killed $d4 def $z4 ; CHECK-NEXT: mov z5.d, d5 ; CHECK-NEXT: mov z4.d, d4 ; CHECK-NEXT: mov z24.d, z6.d ; CHECK-NEXT: mov z7.d, z6.d -; CHECK-NEXT: zip2 z25.d, z4.d, z5.d -; CHECK-NEXT: zip1 z4.d, z4.d, z5.d ; CHECK-NEXT: fcmla z24.d, p0/m, z1.d, z3.d, #0 ; CHECK-NEXT: fcmla z7.d, p0/m, z0.d, z2.d, #0 ; CHECK-NEXT: fcmla z24.d, p0/m, z1.d, z3.d, #90 +; CHECK-NEXT: zip2 z1.d, z4.d, z5.d ; CHECK-NEXT: fcmla z7.d, p0/m, z0.d, z2.d, #90 +; CHECK-NEXT: zip1 z2.d, z4.d, z5.d ; CHECK-NEXT: mov z0.d, z6.d -; CHECK-NEXT: fcmla z6.d, p0/m, z24.d, z25.d, #0 -; CHECK-NEXT: fcmla z0.d, p0/m, z7.d, z4.d, #0 -; CHECK-NEXT: fcmla z6.d, p0/m, z24.d, z25.d, #90 -; CHECK-NEXT: fcmla z0.d, p0/m, z7.d, z4.d, #90 +; CHECK-NEXT: fcmla z6.d, p0/m, z24.d, z1.d, #0 +; CHECK-NEXT: fcmla z0.d, p0/m, z7.d, z2.d, #0 +; CHECK-NEXT: fcmla z6.d, p0/m, z24.d, z1.d, #90 +; CHECK-NEXT: fcmla z0.d, p0/m, z7.d, z2.d, #90 ; CHECK-NEXT: mov z1.d, z6.d ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/AArch64/concat_vector-truncate-combine.ll b/llvm/test/CodeGen/AArch64/concat_vector-truncate-combine.ll index 1c254f9ed935..e6d5a2ac0fd7 100644 --- a/llvm/test/CodeGen/AArch64/concat_vector-truncate-combine.ll +++ b/llvm/test/CodeGen/AArch64/concat_vector-truncate-combine.ll @@ -96,8 +96,8 @@ entry: define void @test_concat_fptrunc_v4f64_to_v4f32(ptr %ptr) #1 { ; CHECK-LABEL: test_concat_fptrunc_v4f64_to_v4f32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: fmov z0.s, #1.00000000 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: st1w { z0.s }, p0, [x0] ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/AArch64/dag-combine-concat-vectors.ll b/llvm/test/CodeGen/AArch64/dag-combine-concat-vectors.ll index 4a2e85c715f7..83c7f73800af 100644 --- a/llvm/test/CodeGen/AArch64/dag-combine-concat-vectors.ll +++ b/llvm/test/CodeGen/AArch64/dag-combine-concat-vectors.ll @@ -9,8 +9,8 @@ define fastcc i8 @allocno_reload_assign() { ; CHECK-LABEL: allocno_reload_assign: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z0.b, #0 // =0x0 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z16.d, #0 // =0x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ptrue p1.b ; CHECK-NEXT: uunpklo z1.h, z0.b ; CHECK-NEXT: uunpkhi z0.h, z0.b @@ -48,12 +48,12 @@ define fastcc i8 @allocno_reload_assign() { ; CHECK-NEXT: punpklo p4.h, p3.b ; CHECK-NEXT: punpkhi p3.h, p3.b ; CHECK-NEXT: st1b { z2.d }, p4, [z16.d] -; CHECK-NEXT: punpklo p4.h, p2.b -; CHECK-NEXT: punpkhi p2.h, p2.b ; CHECK-NEXT: st1b { z3.d }, p3, [z16.d] -; CHECK-NEXT: punpklo p3.h, p4.b -; CHECK-NEXT: st1b { z4.d }, p3, [z16.d] -; CHECK-NEXT: punpkhi p3.h, p4.b +; CHECK-NEXT: punpklo p3.h, p2.b +; CHECK-NEXT: punpkhi p2.h, p2.b +; CHECK-NEXT: punpklo p4.h, p3.b +; CHECK-NEXT: punpkhi p3.h, p3.b +; CHECK-NEXT: st1b { z4.d }, p4, [z16.d] ; CHECK-NEXT: st1b { z5.d }, p3, [z16.d] ; CHECK-NEXT: punpklo p3.h, p2.b ; CHECK-NEXT: punpkhi p2.h, p2.b diff --git a/llvm/test/CodeGen/AArch64/div-rem-pair-recomposition-signed.ll b/llvm/test/CodeGen/AArch64/div-rem-pair-recomposition-signed.ll index 49ad3ae7d629..6e13ae6feb66 100644 --- a/llvm/test/CodeGen/AArch64/div-rem-pair-recomposition-signed.ll +++ b/llvm/test/CodeGen/AArch64/div-rem-pair-recomposition-signed.ll @@ -228,13 +228,13 @@ define <2 x i64> @vector_i128_i64(<2 x i64> %x, <2 x i64> %y, ptr %divdst) nounw ; ALL-NEXT: sdiv x9, x9, x8 ; ALL-NEXT: sdiv x11, x11, x10 ; ALL-NEXT: mul x8, x9, x8 +; ALL-NEXT: fmov d2, x9 ; ALL-NEXT: fmov d1, x8 ; ALL-NEXT: mul x10, x11, x10 +; ALL-NEXT: mov v2.d[1], x11 +; ALL-NEXT: str q2, [x0] ; ALL-NEXT: mov v1.d[1], x10 ; ALL-NEXT: sub v0.2d, v0.2d, v1.2d -; ALL-NEXT: fmov d1, x9 -; ALL-NEXT: mov v1.d[1], x11 -; ALL-NEXT: str q1, [x0] ; ALL-NEXT: ret %div = sdiv <2 x i64> %x, %y store <2 x i64> %div, ptr %divdst, align 16 diff --git a/llvm/test/CodeGen/AArch64/div-rem-pair-recomposition-unsigned.ll b/llvm/test/CodeGen/AArch64/div-rem-pair-recomposition-unsigned.ll index 3bc50b2f03d8..650219e03b8a 100644 --- a/llvm/test/CodeGen/AArch64/div-rem-pair-recomposition-unsigned.ll +++ b/llvm/test/CodeGen/AArch64/div-rem-pair-recomposition-unsigned.ll @@ -228,13 +228,13 @@ define <2 x i64> @vector_i128_i64(<2 x i64> %x, <2 x i64> %y, ptr %divdst) nounw ; ALL-NEXT: udiv x9, x9, x8 ; ALL-NEXT: udiv x11, x11, x10 ; ALL-NEXT: mul x8, x9, x8 +; ALL-NEXT: fmov d2, x9 ; ALL-NEXT: fmov d1, x8 ; ALL-NEXT: mul x10, x11, x10 +; ALL-NEXT: mov v2.d[1], x11 +; ALL-NEXT: str q2, [x0] ; ALL-NEXT: mov v1.d[1], x10 ; ALL-NEXT: sub v0.2d, v0.2d, v1.2d -; ALL-NEXT: fmov d1, x9 -; ALL-NEXT: mov v1.d[1], x11 -; ALL-NEXT: str q1, [x0] ; ALL-NEXT: ret %div = udiv <2 x i64> %x, %y store <2 x i64> %div, ptr %divdst, align 16 diff --git a/llvm/test/CodeGen/AArch64/extbinopload.ll b/llvm/test/CodeGen/AArch64/extbinopload.ll index dff4831330de..bd9d9b99622e 100644 --- a/llvm/test/CodeGen/AArch64/extbinopload.ll +++ b/llvm/test/CodeGen/AArch64/extbinopload.ll @@ -502,9 +502,9 @@ define <16 x i32> @double_bv_4xv4i8_i32(ptr %p, ptr %q, ptr %r, ptr %s, ptr %t, ; CHECK-NEXT: usubl v4.8h, v4.8b, v5.8b ; CHECK-NEXT: ld1 { v6.s }[1], [x7], #4 ; CHECK-NEXT: ld1 { v7.s }[1], [x7] -; CHECK-NEXT: usubl v5.8h, v6.8b, v7.8b ; CHECK-NEXT: shll v0.4s, v4.4h, #16 ; CHECK-NEXT: shll2 v4.4s, v4.8h, #16 +; CHECK-NEXT: usubl v5.8h, v6.8b, v7.8b ; CHECK-NEXT: saddw v0.4s, v0.4s, v1.4h ; CHECK-NEXT: saddw2 v1.4s, v4.4s, v1.8h ; CHECK-NEXT: shll v6.4s, v5.4h, #16 @@ -647,10 +647,10 @@ define <16 x i32> @extrause_load(ptr %p, ptr %q, ptr %r, ptr %s, ptr %z) { ; CHECK: // %bb.0: ; CHECK-NEXT: ldr s1, [x0] ; CHECK-NEXT: add x8, x3, #8 -; CHECK-NEXT: add x11, x3, #12 +; CHECK-NEXT: add x11, x1, #12 ; CHECK-NEXT: str s1, [x4] ; CHECK-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-NEXT: ldp s0, s4, [x2] +; CHECK-NEXT: ldr s0, [x2] ; CHECK-NEXT: ushll v2.8h, v0.8b, #0 ; CHECK-NEXT: umov w9, v2.h[0] ; CHECK-NEXT: umov w10, v2.h[1] @@ -664,31 +664,32 @@ define <16 x i32> @extrause_load(ptr %p, ptr %q, ptr %r, ptr %s, ptr %z) { ; CHECK-NEXT: add x9, x1, #4 ; CHECK-NEXT: mov v1.d[1], v2.d[0] ; CHECK-NEXT: mov v0.b[11], w10 -; CHECK-NEXT: add x10, x1, #12 +; CHECK-NEXT: add x10, x3, #12 ; CHECK-NEXT: bic v1.8h, #255, lsl #8 ; CHECK-NEXT: ld1 { v0.s }[3], [x3], #4 -; CHECK-NEXT: ldr s3, [x0, #12] -; CHECK-NEXT: ldp s2, s7, [x0, #4] -; CHECK-NEXT: ld1 { v4.s }[1], [x3] -; CHECK-NEXT: ldp s5, s6, [x2, #8] -; CHECK-NEXT: ld1 { v3.s }[1], [x10] -; CHECK-NEXT: ld1 { v2.s }[1], [x9] -; CHECK-NEXT: ld1 { v5.s }[1], [x8] -; CHECK-NEXT: ld1 { v6.s }[1], [x11] +; CHECK-NEXT: ldr s5, [x0, #4] +; CHECK-NEXT: ldp s2, s3, [x2, #4] +; CHECK-NEXT: ldr s7, [x2, #12] +; CHECK-NEXT: ldp s6, s4, [x0, #8] +; CHECK-NEXT: ld1 { v5.s }[1], [x9] +; CHECK-NEXT: ld1 { v7.s }[1], [x10] +; CHECK-NEXT: ld1 { v3.s }[1], [x8] +; CHECK-NEXT: ld1 { v2.s }[1], [x3] ; CHECK-NEXT: add x8, x1, #8 -; CHECK-NEXT: ld1 { v7.s }[1], [x8] -; CHECK-NEXT: uaddl v2.8h, v2.8b, v3.8b -; CHECK-NEXT: ushll v3.8h, v5.8b, #0 -; CHECK-NEXT: uaddl v4.8h, v4.8b, v6.8b -; CHECK-NEXT: uaddw v1.8h, v1.8h, v7.8b +; CHECK-NEXT: ld1 { v4.s }[1], [x11] +; CHECK-NEXT: ld1 { v6.s }[1], [x8] +; CHECK-NEXT: ushll v3.8h, v3.8b, #0 +; CHECK-NEXT: uaddl v2.8h, v2.8b, v7.8b +; CHECK-NEXT: uaddl v4.8h, v5.8b, v4.8b +; CHECK-NEXT: uaddw v1.8h, v1.8h, v6.8b ; CHECK-NEXT: uaddw2 v5.8h, v3.8h, v0.16b -; CHECK-NEXT: ushll v0.4s, v2.4h, #3 +; CHECK-NEXT: ushll v6.4s, v2.4h, #3 ; CHECK-NEXT: ushll2 v2.4s, v2.8h, #3 -; CHECK-NEXT: ushll v6.4s, v4.4h, #3 +; CHECK-NEXT: ushll v0.4s, v4.4h, #3 ; CHECK-NEXT: ushll2 v3.4s, v4.8h, #3 ; CHECK-NEXT: uaddw v0.4s, v0.4s, v1.4h -; CHECK-NEXT: uaddw2 v1.4s, v2.4s, v1.8h -; CHECK-NEXT: uaddw2 v3.4s, v3.4s, v5.8h +; CHECK-NEXT: uaddw2 v1.4s, v3.4s, v1.8h +; CHECK-NEXT: uaddw2 v3.4s, v2.4s, v5.8h ; CHECK-NEXT: uaddw v2.4s, v6.4s, v5.4h ; CHECK-NEXT: ret %lp1 = load <4 x i8>, ptr %p @@ -762,35 +763,35 @@ define <16 x i32> @extrause_shuffle(ptr %p, ptr %q, ptr %r, ptr %s, ptr %z) { ; CHECK-NEXT: add x8, x1, #8 ; CHECK-NEXT: ldr s6, [x1, #12] ; CHECK-NEXT: ldp s17, s18, [x2, #8] -; CHECK-NEXT: ldp s2, s3, [x2] +; CHECK-NEXT: ldp s3, s5, [x2] ; CHECK-NEXT: add x9, x3, #8 ; CHECK-NEXT: mov v4.16b, v1.16b ; CHECK-NEXT: ldp s7, s16, [x0] -; CHECK-NEXT: ldr s5, [x3, #12] +; CHECK-NEXT: ldr s2, [x3, #12] ; CHECK-NEXT: mov v1.s[1], v6.s[0] -; CHECK-NEXT: ld1 { v2.s }[1], [x3], #4 +; CHECK-NEXT: ld1 { v3.s }[1], [x3], #4 ; CHECK-NEXT: mov v4.s[1], v6.s[0] ; CHECK-NEXT: ld1 { v7.s }[1], [x1], #4 ; CHECK-NEXT: ld1 { v16.s }[1], [x1] -; CHECK-NEXT: ld1 { v3.s }[1], [x3] -; CHECK-NEXT: ld1 { v0.s }[1], [x8] +; CHECK-NEXT: ld1 { v5.s }[1], [x3] ; CHECK-NEXT: ld1 { v17.s }[1], [x9] +; CHECK-NEXT: ld1 { v0.s }[1], [x8] ; CHECK-NEXT: mov v4.s[2], v18.s[0] -; CHECK-NEXT: mov v18.s[1], v5.s[0] +; CHECK-NEXT: mov v18.s[1], v2.s[0] ; CHECK-NEXT: uaddl v1.8h, v16.8b, v1.8b ; CHECK-NEXT: uaddl v6.8h, v7.8b, v0.8b -; CHECK-NEXT: uaddl v2.8h, v2.8b, v17.8b -; CHECK-NEXT: uaddl v3.8h, v3.8b, v18.8b +; CHECK-NEXT: uaddl v7.8h, v3.8b, v17.8b ; CHECK-NEXT: ushll v0.4s, v1.4h, #3 ; CHECK-NEXT: ushll2 v1.4s, v1.8h, #3 -; CHECK-NEXT: mov v4.s[3], v5.s[0] +; CHECK-NEXT: uaddl v5.8h, v5.8b, v18.8b +; CHECK-NEXT: mov v4.s[3], v2.s[0] ; CHECK-NEXT: uaddw v0.4s, v0.4s, v6.4h ; CHECK-NEXT: uaddw2 v1.4s, v1.4s, v6.8h -; CHECK-NEXT: ushll v7.4s, v3.4h, #3 -; CHECK-NEXT: ushll2 v3.4s, v3.8h, #3 +; CHECK-NEXT: ushll v16.4s, v5.4h, #3 +; CHECK-NEXT: ushll2 v3.4s, v5.8h, #3 ; CHECK-NEXT: str q4, [x4] -; CHECK-NEXT: uaddw2 v3.4s, v3.4s, v2.8h -; CHECK-NEXT: uaddw v2.4s, v7.4s, v2.4h +; CHECK-NEXT: uaddw2 v3.4s, v3.4s, v7.8h +; CHECK-NEXT: uaddw v2.4s, v16.4s, v7.4h ; CHECK-NEXT: ret %lp1 = load <4 x i8>, ptr %p %p2 = getelementptr i8, ptr %p, i32 4 @@ -873,8 +874,8 @@ define <16 x i32> @extrause_ext(ptr %p, ptr %q, ptr %r, ptr %s, ptr %z) { ; CHECK-NEXT: ld1 { v4.s }[1], [x11] ; CHECK-NEXT: ld1 { v2.s }[1], [x3] ; CHECK-NEXT: ld1 { v0.s }[1], [x10] -; CHECK-NEXT: ld1 { v7.s }[1], [x9] ; CHECK-NEXT: ld1 { v6.s }[1], [x8] +; CHECK-NEXT: ld1 { v7.s }[1], [x9] ; CHECK-NEXT: uaddl v5.8h, v5.8b, v4.8b ; CHECK-NEXT: uaddl v2.8h, v2.8b, v0.8b ; CHECK-NEXT: ushll v16.8h, v0.8b, #0 @@ -972,8 +973,8 @@ define <16 x i32> @extrause_add(ptr %p, ptr %q, ptr %r, ptr %s, ptr %z) { ; CHECK-NEXT: ld1 { v5.s }[1], [x11] ; CHECK-NEXT: ld1 { v3.s }[1], [x3] ; CHECK-NEXT: ld1 { v7.s }[1], [x10] -; CHECK-NEXT: ld1 { v4.s }[1], [x9] ; CHECK-NEXT: ld1 { v6.s }[1], [x8] +; CHECK-NEXT: ld1 { v4.s }[1], [x9] ; CHECK-NEXT: uaddl v5.8h, v1.8b, v5.8b ; CHECK-NEXT: uaddl v7.8h, v3.8b, v7.8b ; CHECK-NEXT: uaddl v1.8h, v0.8b, v4.8b @@ -1072,23 +1073,23 @@ define <16 x i32> @extrause_ext2(ptr %p, ptr %q, ptr %r, ptr %s, ptr %z) { ; CHECK-NEXT: ld1 { v6.s }[1], [x9] ; CHECK-NEXT: ld1 { v4.s }[1], [x8] ; CHECK-NEXT: uaddl v7.8h, v3.8b, v7.8b -; CHECK-NEXT: uaddl v3.8h, v1.8b, v5.8b +; CHECK-NEXT: uaddl v1.8h, v1.8b, v5.8b ; CHECK-NEXT: uaddl v2.8h, v2.8b, v6.8b ; CHECK-NEXT: uaddl v4.8h, v0.8b, v4.8b ; CHECK-NEXT: ushll v0.4s, v7.4h, #3 -; CHECK-NEXT: ushll2 v1.4s, v7.8h, #3 -; CHECK-NEXT: ushll v5.4s, v3.4h, #3 -; CHECK-NEXT: ushll2 v6.4s, v3.8h, #3 -; CHECK-NEXT: ushll2 v16.4s, v3.8h, #0 -; CHECK-NEXT: ushll v17.4s, v3.4h, #0 -; CHECK-NEXT: uaddw2 v1.4s, v1.4s, v2.8h +; CHECK-NEXT: ushll2 v3.4s, v7.8h, #3 +; CHECK-NEXT: ushll v6.4s, v1.4h, #3 +; CHECK-NEXT: ushll2 v16.4s, v1.8h, #3 +; CHECK-NEXT: ushll2 v5.4s, v1.8h, #0 +; CHECK-NEXT: ushll v17.4s, v1.4h, #0 +; CHECK-NEXT: ushll2 v18.4s, v7.8h, #0 +; CHECK-NEXT: uaddw2 v1.4s, v3.4s, v2.8h ; CHECK-NEXT: uaddw v0.4s, v0.4s, v2.4h -; CHECK-NEXT: uaddw v2.4s, v5.4s, v4.4h -; CHECK-NEXT: uaddw2 v3.4s, v6.4s, v4.8h -; CHECK-NEXT: ushll2 v4.4s, v7.8h, #0 -; CHECK-NEXT: ushll v5.4s, v7.4h, #0 -; CHECK-NEXT: stp q17, q16, [x4, #32] -; CHECK-NEXT: stp q5, q4, [x4] +; CHECK-NEXT: uaddw v2.4s, v6.4s, v4.4h +; CHECK-NEXT: uaddw2 v3.4s, v16.4s, v4.8h +; CHECK-NEXT: ushll v4.4s, v7.4h, #0 +; CHECK-NEXT: stp q17, q5, [x4, #32] +; CHECK-NEXT: stp q4, q18, [x4] ; CHECK-NEXT: ret %lp1 = load <4 x i8>, ptr %p %p2 = getelementptr i8, ptr %p, i32 4 @@ -1157,32 +1158,32 @@ define <16 x i32> @extrause_ext2(ptr %p, ptr %q, ptr %r, ptr %s, ptr %z) { define <16 x i32> @extrause_shl(ptr %p, ptr %q, ptr %r, ptr %s, ptr %z) { ; CHECK-LABEL: extrause_shl: ; CHECK: // %bb.0: -; CHECK-NEXT: ldp s0, s1, [x0] +; CHECK-NEXT: ldp s1, s2, [x0] ; CHECK-NEXT: add x10, x3, #12 -; CHECK-NEXT: ldp s2, s3, [x2] +; CHECK-NEXT: ldp s0, s3, [x2] ; CHECK-NEXT: add x11, x1, #12 ; CHECK-NEXT: ldp s4, s5, [x0, #8] ; CHECK-NEXT: add x8, x3, #8 ; CHECK-NEXT: ldp s6, s7, [x2, #8] ; CHECK-NEXT: add x9, x1, #8 -; CHECK-NEXT: ld1 { v2.s }[1], [x3], #4 -; CHECK-NEXT: ld1 { v0.s }[1], [x1], #4 -; CHECK-NEXT: ld1 { v1.s }[1], [x1] +; CHECK-NEXT: ld1 { v0.s }[1], [x3], #4 +; CHECK-NEXT: ld1 { v1.s }[1], [x1], #4 +; CHECK-NEXT: ld1 { v2.s }[1], [x1] ; CHECK-NEXT: ld1 { v5.s }[1], [x11] ; CHECK-NEXT: ld1 { v3.s }[1], [x3] ; CHECK-NEXT: ld1 { v7.s }[1], [x10] ; CHECK-NEXT: ld1 { v4.s }[1], [x9] ; CHECK-NEXT: ld1 { v6.s }[1], [x8] -; CHECK-NEXT: uaddl v1.8h, v1.8b, v5.8b +; CHECK-NEXT: uaddl v2.8h, v2.8b, v5.8b ; CHECK-NEXT: uaddl v3.8h, v3.8b, v7.8b -; CHECK-NEXT: uaddl v4.8h, v0.8b, v4.8b -; CHECK-NEXT: uaddl v2.8h, v2.8b, v6.8b -; CHECK-NEXT: ushll v5.4s, v1.4h, #3 +; CHECK-NEXT: uaddl v4.8h, v1.8b, v4.8b +; CHECK-NEXT: ushll v5.4s, v2.4h, #3 +; CHECK-NEXT: ushll2 v7.4s, v2.8h, #3 +; CHECK-NEXT: uaddl v2.8h, v0.8b, v6.8b ; CHECK-NEXT: ushll v6.4s, v3.4h, #3 -; CHECK-NEXT: ushll2 v7.4s, v1.8h, #3 ; CHECK-NEXT: ushll2 v16.4s, v3.8h, #3 -; CHECK-NEXT: uaddw v0.4s, v5.4s, v4.4h ; CHECK-NEXT: uaddw2 v1.4s, v7.4s, v4.8h +; CHECK-NEXT: uaddw v0.4s, v5.4s, v4.4h ; CHECK-NEXT: stp q5, q7, [x4] ; CHECK-NEXT: uaddw2 v3.4s, v16.4s, v2.8h ; CHECK-NEXT: uaddw v2.4s, v6.4s, v2.4h diff --git a/llvm/test/CodeGen/AArch64/fcmp.ll b/llvm/test/CodeGen/AArch64/fcmp.ll index 9916aeeab1ca..b1ca88975a62 100644 --- a/llvm/test/CodeGen/AArch64/fcmp.ll +++ b/llvm/test/CodeGen/AArch64/fcmp.ll @@ -280,10 +280,10 @@ define <3 x i32> @v3f64_i32(<3 x double> %a, <3 x double> %b, <3 x i32> %d, <3 x ; CHECK-GI-NEXT: uzp1 v0.4s, v0.4s, v2.4s ; CHECK-GI-NEXT: fmov s2, w8 ; CHECK-GI-NEXT: mov v2.s[1], w8 -; CHECK-GI-NEXT: neg v3.4s, v1.4s ; CHECK-GI-NEXT: ushl v0.4s, v0.4s, v1.4s +; CHECK-GI-NEXT: neg v1.4s, v1.4s ; CHECK-GI-NEXT: mov v2.s[2], w8 -; CHECK-GI-NEXT: sshl v0.4s, v0.4s, v3.4s +; CHECK-GI-NEXT: sshl v0.4s, v0.4s, v1.4s ; CHECK-GI-NEXT: eor v1.16b, v0.16b, v2.16b ; CHECK-GI-NEXT: and v0.16b, v6.16b, v0.16b ; CHECK-GI-NEXT: and v1.16b, v7.16b, v1.16b @@ -348,10 +348,10 @@ define <3 x float> @v3f32_float(<3 x float> %a, <3 x float> %b, <3 x float> %d, ; CHECK-GI-NEXT: mov w8, #-1 // =0xffffffff ; CHECK-GI-NEXT: fmov s1, w8 ; CHECK-GI-NEXT: mov v1.s[1], w8 -; CHECK-GI-NEXT: neg v5.4s, v4.4s ; CHECK-GI-NEXT: ushl v0.4s, v0.4s, v4.4s +; CHECK-GI-NEXT: neg v4.4s, v4.4s +; CHECK-GI-NEXT: sshl v0.4s, v0.4s, v4.4s ; CHECK-GI-NEXT: mov v1.s[2], w8 -; CHECK-GI-NEXT: sshl v0.4s, v0.4s, v5.4s ; CHECK-GI-NEXT: eor v1.16b, v0.16b, v1.16b ; CHECK-GI-NEXT: and v0.16b, v2.16b, v0.16b ; CHECK-GI-NEXT: and v1.16b, v3.16b, v1.16b @@ -426,10 +426,10 @@ define <3 x i32> @v3f32_i32(<3 x float> %a, <3 x float> %b, <3 x i32> %d, <3 x i ; CHECK-GI-NEXT: mov w8, #-1 // =0xffffffff ; CHECK-GI-NEXT: fmov s1, w8 ; CHECK-GI-NEXT: mov v1.s[1], w8 -; CHECK-GI-NEXT: neg v5.4s, v4.4s ; CHECK-GI-NEXT: ushl v0.4s, v0.4s, v4.4s +; CHECK-GI-NEXT: neg v4.4s, v4.4s +; CHECK-GI-NEXT: sshl v0.4s, v0.4s, v4.4s ; CHECK-GI-NEXT: mov v1.s[2], w8 -; CHECK-GI-NEXT: sshl v0.4s, v0.4s, v5.4s ; CHECK-GI-NEXT: eor v1.16b, v0.16b, v1.16b ; CHECK-GI-NEXT: and v0.16b, v2.16b, v0.16b ; CHECK-GI-NEXT: and v1.16b, v3.16b, v1.16b @@ -545,8 +545,8 @@ define <7 x half> @v7f16_half(<7 x half> %a, <7 x half> %b, <7 x half> %d, <7 x ; CHECK-GI-NOFP16-LABEL: v7f16_half: ; CHECK-GI-NOFP16: // %bb.0: // %entry ; CHECK-GI-NOFP16-NEXT: mov w8, #15 // =0xf -; CHECK-GI-NOFP16-NEXT: mov h6, v0.h[4] -; CHECK-GI-NOFP16-NEXT: mov h7, v0.h[5] +; CHECK-GI-NOFP16-NEXT: mov h5, v0.h[4] +; CHECK-GI-NOFP16-NEXT: mov h6, v0.h[5] ; CHECK-GI-NOFP16-NEXT: fmov s4, w8 ; CHECK-GI-NOFP16-NEXT: mov h16, v1.h[4] ; CHECK-GI-NOFP16-NEXT: mov w8, #65535 // =0xffff @@ -555,32 +555,32 @@ define <7 x half> @v7f16_half(<7 x half> %a, <7 x half> %b, <7 x half> %d, <7 x ; CHECK-GI-NOFP16-NEXT: mov h19, v1.h[6] ; CHECK-GI-NOFP16-NEXT: fcvtl v0.4s, v0.4h ; CHECK-GI-NOFP16-NEXT: fcvtl v1.4s, v1.4h -; CHECK-GI-NOFP16-NEXT: mov v5.16b, v4.16b -; CHECK-GI-NOFP16-NEXT: mov v6.h[1], v7.h[0] -; CHECK-GI-NOFP16-NEXT: fmov s7, w8 +; CHECK-GI-NOFP16-NEXT: mov v7.16b, v4.16b +; CHECK-GI-NOFP16-NEXT: mov v5.h[1], v6.h[0] +; CHECK-GI-NOFP16-NEXT: fmov s6, w8 ; CHECK-GI-NOFP16-NEXT: mov v16.h[1], v17.h[0] -; CHECK-GI-NOFP16-NEXT: mov v5.h[1], v4.h[0] -; CHECK-GI-NOFP16-NEXT: mov v17.16b, v7.16b +; CHECK-GI-NOFP16-NEXT: mov v7.h[1], v4.h[0] +; CHECK-GI-NOFP16-NEXT: mov v17.16b, v6.16b ; CHECK-GI-NOFP16-NEXT: fcmgt v0.4s, v1.4s, v0.4s -; CHECK-GI-NOFP16-NEXT: mov v6.h[2], v18.h[0] -; CHECK-GI-NOFP16-NEXT: mov v17.h[1], v7.h[0] +; CHECK-GI-NOFP16-NEXT: mov v5.h[2], v18.h[0] +; CHECK-GI-NOFP16-NEXT: mov v17.h[1], v6.h[0] ; CHECK-GI-NOFP16-NEXT: mov v16.h[2], v19.h[0] -; CHECK-GI-NOFP16-NEXT: mov v5.h[2], v4.h[0] -; CHECK-GI-NOFP16-NEXT: fcvtl v6.4s, v6.4h -; CHECK-GI-NOFP16-NEXT: mov v17.h[2], v7.h[0] +; CHECK-GI-NOFP16-NEXT: mov v7.h[2], v4.h[0] +; CHECK-GI-NOFP16-NEXT: fcvtl v5.4s, v5.4h +; CHECK-GI-NOFP16-NEXT: mov v17.h[2], v6.h[0] ; CHECK-GI-NOFP16-NEXT: fcvtl v16.4s, v16.4h -; CHECK-GI-NOFP16-NEXT: mov v5.h[3], v4.h[0] -; CHECK-GI-NOFP16-NEXT: mov v17.h[3], v7.h[0] -; CHECK-GI-NOFP16-NEXT: fcmgt v1.4s, v16.4s, v6.4s -; CHECK-GI-NOFP16-NEXT: mov v5.h[4], v4.h[0] -; CHECK-GI-NOFP16-NEXT: mov v17.h[4], v7.h[0] +; CHECK-GI-NOFP16-NEXT: mov v7.h[3], v4.h[0] +; CHECK-GI-NOFP16-NEXT: mov v17.h[3], v6.h[0] +; CHECK-GI-NOFP16-NEXT: fcmgt v1.4s, v16.4s, v5.4s +; CHECK-GI-NOFP16-NEXT: mov v7.h[4], v4.h[0] +; CHECK-GI-NOFP16-NEXT: mov v17.h[4], v6.h[0] ; CHECK-GI-NOFP16-NEXT: uzp1 v0.8h, v0.8h, v1.8h -; CHECK-GI-NOFP16-NEXT: mov v5.h[5], v4.h[0] -; CHECK-GI-NOFP16-NEXT: mov v17.h[5], v7.h[0] -; CHECK-GI-NOFP16-NEXT: mov v5.h[6], v4.h[0] -; CHECK-GI-NOFP16-NEXT: mov v17.h[6], v7.h[0] -; CHECK-GI-NOFP16-NEXT: neg v1.8h, v5.8h -; CHECK-GI-NOFP16-NEXT: ushl v0.8h, v0.8h, v5.8h +; CHECK-GI-NOFP16-NEXT: mov v7.h[5], v4.h[0] +; CHECK-GI-NOFP16-NEXT: mov v17.h[5], v6.h[0] +; CHECK-GI-NOFP16-NEXT: mov v7.h[6], v4.h[0] +; CHECK-GI-NOFP16-NEXT: mov v17.h[6], v6.h[0] +; CHECK-GI-NOFP16-NEXT: ushl v0.8h, v0.8h, v7.8h +; CHECK-GI-NOFP16-NEXT: neg v1.8h, v7.8h ; CHECK-GI-NOFP16-NEXT: sshl v0.8h, v0.8h, v1.8h ; CHECK-GI-NOFP16-NEXT: eor v1.16b, v0.16b, v17.16b ; CHECK-GI-NOFP16-NEXT: and v0.16b, v2.16b, v0.16b @@ -609,8 +609,8 @@ define <7 x half> @v7f16_half(<7 x half> %a, <7 x half> %b, <7 x half> %d, <7 x ; CHECK-GI-FP16-NEXT: mov v7.h[5], v6.h[0] ; CHECK-GI-FP16-NEXT: mov v5.h[6], v4.h[0] ; CHECK-GI-FP16-NEXT: mov v7.h[6], v6.h[0] -; CHECK-GI-FP16-NEXT: neg v1.8h, v5.8h ; CHECK-GI-FP16-NEXT: ushl v0.8h, v0.8h, v5.8h +; CHECK-GI-FP16-NEXT: neg v1.8h, v5.8h ; CHECK-GI-FP16-NEXT: sshl v0.8h, v0.8h, v1.8h ; CHECK-GI-FP16-NEXT: eor v1.16b, v0.16b, v7.16b ; CHECK-GI-FP16-NEXT: and v0.16b, v2.16b, v0.16b @@ -1047,6 +1047,7 @@ define <7 x i32> @v7f16_i32(<7 x half> %a, <7 x half> %b, <7 x i32> %d, <7 x i32 ; CHECK-GI-NOFP16-NEXT: mov h6, v0.h[6] ; CHECK-GI-NOFP16-NEXT: mov h7, v1.h[6] ; CHECK-GI-NOFP16-NEXT: fmov s16, w0 +; CHECK-GI-NOFP16-NEXT: fmov s18, w4 ; CHECK-GI-NOFP16-NEXT: fcvtl v0.4s, v0.4h ; CHECK-GI-NOFP16-NEXT: fcvtl v1.4s, v1.4h ; CHECK-GI-NOFP16-NEXT: mov v2.h[1], v3.h[0] @@ -1054,6 +1055,7 @@ define <7 x i32> @v7f16_i32(<7 x half> %a, <7 x half> %b, <7 x i32> %d, <7 x i32 ; CHECK-GI-NOFP16-NEXT: mov v4.h[1], v5.h[0] ; CHECK-GI-NOFP16-NEXT: ldr s5, [sp] ; CHECK-GI-NOFP16-NEXT: mov v16.s[1], w1 +; CHECK-GI-NOFP16-NEXT: mov v18.s[1], w5 ; CHECK-GI-NOFP16-NEXT: mov v3.s[1], w8 ; CHECK-GI-NOFP16-NEXT: fmov w9, s5 ; CHECK-GI-NOFP16-NEXT: fmov s5, w7 @@ -1069,27 +1071,25 @@ define <7 x i32> @v7f16_i32(<7 x half> %a, <7 x half> %b, <7 x i32> %d, <7 x i32 ; CHECK-GI-NOFP16-NEXT: mov v3.s[2], w8 ; CHECK-GI-NOFP16-NEXT: mov w8, #-1 // =0xffffffff ; CHECK-GI-NOFP16-NEXT: mov v7.s[1], v17.s[0] -; CHECK-GI-NOFP16-NEXT: fcvtl v2.4s, v2.4h ; CHECK-GI-NOFP16-NEXT: ldr s17, [sp, #40] +; CHECK-GI-NOFP16-NEXT: fcvtl v2.4s, v2.4h +; CHECK-GI-NOFP16-NEXT: mov v18.s[2], w6 ; CHECK-GI-NOFP16-NEXT: fcvtl v4.4s, v4.4h ; CHECK-GI-NOFP16-NEXT: mov v16.s[3], w3 ; CHECK-GI-NOFP16-NEXT: mov v5.s[2], w9 -; CHECK-GI-NOFP16-NEXT: neg v18.4s, v3.4s ; CHECK-GI-NOFP16-NEXT: mov v7.s[2], v17.s[0] ; CHECK-GI-NOFP16-NEXT: fcmgt v2.4s, v4.4s, v2.4s ; CHECK-GI-NOFP16-NEXT: fmov s4, w8 ; CHECK-GI-NOFP16-NEXT: mov v4.s[1], w8 ; CHECK-GI-NOFP16-NEXT: ushl v2.4s, v2.4s, v3.4s -; CHECK-GI-NOFP16-NEXT: fmov s3, w4 -; CHECK-GI-NOFP16-NEXT: mov v3.s[1], w5 +; CHECK-GI-NOFP16-NEXT: neg v3.4s, v3.4s ; CHECK-GI-NOFP16-NEXT: mov v4.s[2], w8 -; CHECK-GI-NOFP16-NEXT: sshl v2.4s, v2.4s, v18.4s +; CHECK-GI-NOFP16-NEXT: sshl v2.4s, v2.4s, v3.4s ; CHECK-GI-NOFP16-NEXT: fmov w8, s6 -; CHECK-GI-NOFP16-NEXT: mov v3.s[2], w6 -; CHECK-GI-NOFP16-NEXT: eor v1.16b, v2.16b, v4.16b ; CHECK-GI-NOFP16-NEXT: mov v5.s[3], w8 +; CHECK-GI-NOFP16-NEXT: eor v1.16b, v2.16b, v4.16b +; CHECK-GI-NOFP16-NEXT: and v2.16b, v18.16b, v2.16b ; CHECK-GI-NOFP16-NEXT: and v1.16b, v7.16b, v1.16b -; CHECK-GI-NOFP16-NEXT: and v2.16b, v3.16b, v2.16b ; CHECK-GI-NOFP16-NEXT: bsl v0.16b, v16.16b, v5.16b ; CHECK-GI-NOFP16-NEXT: orr v1.16b, v2.16b, v1.16b ; CHECK-GI-NOFP16-NEXT: mov s2, v0.s[1] @@ -1111,30 +1111,32 @@ define <7 x i32> @v7f16_i32(<7 x half> %a, <7 x half> %b, <7 x i32> %d, <7 x i32 ; CHECK-GI-FP16-NEXT: fcmgt v0.8h, v1.8h, v0.8h ; CHECK-GI-FP16-NEXT: mov w10, #31 // =0x1f ; CHECK-GI-FP16-NEXT: ldr s3, [sp] -; CHECK-GI-FP16-NEXT: fmov s1, w10 +; CHECK-GI-FP16-NEXT: fmov s2, w10 ; CHECK-GI-FP16-NEXT: fmov s6, w0 ; CHECK-GI-FP16-NEXT: ldr s4, [sp, #8] +; CHECK-GI-FP16-NEXT: fmov s17, w4 ; CHECK-GI-FP16-NEXT: ldr s7, [sp, #24] ; CHECK-GI-FP16-NEXT: ldr s16, [sp, #32] ; CHECK-GI-FP16-NEXT: umov w8, v0.h[4] ; CHECK-GI-FP16-NEXT: umov w9, v0.h[5] -; CHECK-GI-FP16-NEXT: mov v1.s[1], w10 +; CHECK-GI-FP16-NEXT: mov v2.s[1], w10 ; CHECK-GI-FP16-NEXT: mov v6.s[1], w1 +; CHECK-GI-FP16-NEXT: mov v17.s[1], w5 ; CHECK-GI-FP16-NEXT: mov v7.s[1], v16.s[0] ; CHECK-GI-FP16-NEXT: ldr s16, [sp, #40] -; CHECK-GI-FP16-NEXT: fmov s2, w8 +; CHECK-GI-FP16-NEXT: fmov s1, w8 ; CHECK-GI-FP16-NEXT: umov w8, v0.h[6] -; CHECK-GI-FP16-NEXT: mov v1.s[2], w10 +; CHECK-GI-FP16-NEXT: mov v2.s[2], w10 ; CHECK-GI-FP16-NEXT: ushll v0.4s, v0.4h, #0 ; CHECK-GI-FP16-NEXT: mov v6.s[2], w2 +; CHECK-GI-FP16-NEXT: mov v17.s[2], w6 ; CHECK-GI-FP16-NEXT: mov v7.s[2], v16.s[0] -; CHECK-GI-FP16-NEXT: mov v2.s[1], w9 +; CHECK-GI-FP16-NEXT: mov v1.s[1], w9 ; CHECK-GI-FP16-NEXT: mov w9, #-1 // =0xffffffff ; CHECK-GI-FP16-NEXT: fmov s5, w9 -; CHECK-GI-FP16-NEXT: neg v17.4s, v1.4s ; CHECK-GI-FP16-NEXT: shl v0.4s, v0.4s, #31 ; CHECK-GI-FP16-NEXT: mov v6.s[3], w3 -; CHECK-GI-FP16-NEXT: mov v2.s[2], w8 +; CHECK-GI-FP16-NEXT: mov v1.s[2], w8 ; CHECK-GI-FP16-NEXT: fmov w8, s3 ; CHECK-GI-FP16-NEXT: fmov s3, w7 ; CHECK-GI-FP16-NEXT: mov v5.s[1], w9 @@ -1142,26 +1144,24 @@ define <7 x i32> @v7f16_i32(<7 x half> %a, <7 x half> %b, <7 x i32> %d, <7 x i32 ; CHECK-GI-FP16-NEXT: mov v3.s[1], w8 ; CHECK-GI-FP16-NEXT: fmov w8, s4 ; CHECK-GI-FP16-NEXT: ldr s4, [sp, #16] -; CHECK-GI-FP16-NEXT: ushl v1.4s, v2.4s, v1.4s -; CHECK-GI-FP16-NEXT: fmov s2, w4 +; CHECK-GI-FP16-NEXT: ushl v1.4s, v1.4s, v2.4s +; CHECK-GI-FP16-NEXT: neg v2.4s, v2.4s ; CHECK-GI-FP16-NEXT: mov v5.s[2], w9 -; CHECK-GI-FP16-NEXT: mov v2.s[1], w5 ; CHECK-GI-FP16-NEXT: mov v3.s[2], w8 -; CHECK-GI-FP16-NEXT: sshl v1.4s, v1.4s, v17.4s +; CHECK-GI-FP16-NEXT: sshl v1.4s, v1.4s, v2.4s ; CHECK-GI-FP16-NEXT: fmov w8, s4 -; CHECK-GI-FP16-NEXT: eor v4.16b, v1.16b, v5.16b -; CHECK-GI-FP16-NEXT: mov v2.s[2], w6 +; CHECK-GI-FP16-NEXT: eor v2.16b, v1.16b, v5.16b +; CHECK-GI-FP16-NEXT: and v1.16b, v17.16b, v1.16b ; CHECK-GI-FP16-NEXT: mov v3.s[3], w8 -; CHECK-GI-FP16-NEXT: and v1.16b, v2.16b, v1.16b -; CHECK-GI-FP16-NEXT: and v2.16b, v7.16b, v4.16b +; CHECK-GI-FP16-NEXT: and v2.16b, v7.16b, v2.16b ; CHECK-GI-FP16-NEXT: bsl v0.16b, v6.16b, v3.16b ; CHECK-GI-FP16-NEXT: orr v1.16b, v1.16b, v2.16b ; CHECK-GI-FP16-NEXT: mov s2, v0.s[1] ; CHECK-GI-FP16-NEXT: mov s3, v0.s[2] ; CHECK-GI-FP16-NEXT: mov s4, v0.s[3] -; CHECK-GI-FP16-NEXT: fmov w0, s0 ; CHECK-GI-FP16-NEXT: mov s5, v1.s[1] ; CHECK-GI-FP16-NEXT: mov s6, v1.s[2] +; CHECK-GI-FP16-NEXT: fmov w0, s0 ; CHECK-GI-FP16-NEXT: fmov w4, s1 ; CHECK-GI-FP16-NEXT: fmov w1, s2 ; CHECK-GI-FP16-NEXT: fmov w2, s3 diff --git a/llvm/test/CodeGen/AArch64/fdiv-combine.ll b/llvm/test/CodeGen/AArch64/fdiv-combine.ll index 1ed63f3ef250..0627250d0779 100644 --- a/llvm/test/CodeGen/AArch64/fdiv-combine.ll +++ b/llvm/test/CodeGen/AArch64/fdiv-combine.ll @@ -171,8 +171,8 @@ entry: define @splat_fdiv_nxv2f64(double %D, %a) #1 { ; CHECK-LABEL: splat_fdiv_nxv2f64: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z0.d, d0 ; CHECK-NEXT: fdivr z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/fold-int-pow2-with-fmul-or-fdiv.ll b/llvm/test/CodeGen/AArch64/fold-int-pow2-with-fmul-or-fdiv.ll index 03e64f8b785b..a78addc49008 100644 --- a/llvm/test/CodeGen/AArch64/fold-int-pow2-with-fmul-or-fdiv.ll +++ b/llvm/test/CodeGen/AArch64/fold-int-pow2-with-fmul-or-fdiv.ll @@ -604,8 +604,8 @@ define fastcc i1 @quantum_hadamard(i32 %0) { define @fdiv_pow2_nx4xfloat( %i) "target-features"="+sve" { ; CHECK-LABEL: fdiv_pow2_nx4xfloat: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, #1 // =0x1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: lslr z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: fmov z1.s, #9.00000000 ; CHECK-NEXT: ucvtf z0.s, p0/m, z0.s diff --git a/llvm/test/CodeGen/AArch64/fp-veclib-expansion.ll b/llvm/test/CodeGen/AArch64/fp-veclib-expansion.ll index 67c056c780cc..2c8e2190f820 100644 --- a/llvm/test/CodeGen/AArch64/fp-veclib-expansion.ll +++ b/llvm/test/CodeGen/AArch64/fp-veclib-expansion.ll @@ -62,9 +62,9 @@ define @frem_nxv4f32( %unused, @frem_nxv4f32( %unused, @frem_strict_nxv2f64( %unused, ; ARMPL-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill ; ARMPL-NEXT: .cfi_def_cfa_offset 16 ; ARMPL-NEXT: .cfi_offset w30, -16 -; ARMPL-NEXT: ptrue p0.d ; ARMPL-NEXT: mov z0.d, z1.d ; ARMPL-NEXT: mov z1.d, z2.d +; ARMPL-NEXT: ptrue p0.d ; ARMPL-NEXT: bl armpl_svfmod_f64_x ; ARMPL-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload ; ARMPL-NEXT: ret @@ -102,9 +102,9 @@ define @frem_strict_nxv2f64( %unused, ; SLEEF-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill ; SLEEF-NEXT: .cfi_def_cfa_offset 16 ; SLEEF-NEXT: .cfi_offset w30, -16 -; SLEEF-NEXT: ptrue p0.d ; SLEEF-NEXT: mov z0.d, z1.d ; SLEEF-NEXT: mov z1.d, z2.d +; SLEEF-NEXT: ptrue p0.d ; SLEEF-NEXT: bl _ZGVsMxvv_fmod ; SLEEF-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload ; SLEEF-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll b/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll index 301d28fd7be5..2ea581359af6 100644 --- a/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll +++ b/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll @@ -194,10 +194,10 @@ define <2 x i16> @ustest_f64i16(<2 x double> %x) { ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: fcvtzs v0.2d, v0.2d ; CHECK-NEXT: movi d1, #0x00ffff0000ffff +; CHECK-NEXT: movi v2.2d, #0000000000000000 ; CHECK-NEXT: xtn v0.2s, v0.2d ; CHECK-NEXT: smin v0.2s, v0.2s, v1.2s -; CHECK-NEXT: movi v1.2d, #0000000000000000 -; CHECK-NEXT: smax v0.2s, v0.2s, v1.2s +; CHECK-NEXT: smax v0.2s, v0.2s, v2.2s ; CHECK-NEXT: ret entry: %conv = fptosi <2 x double> %x to <2 x i32> @@ -833,10 +833,10 @@ define <2 x i16> @ustest_f64i16_mm(<2 x double> %x) { ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: fcvtzs v0.2d, v0.2d ; CHECK-NEXT: movi d1, #0x00ffff0000ffff +; CHECK-NEXT: movi v2.2d, #0000000000000000 ; CHECK-NEXT: xtn v0.2s, v0.2d ; CHECK-NEXT: smin v0.2s, v0.2s, v1.2s -; CHECK-NEXT: movi v1.2d, #0000000000000000 -; CHECK-NEXT: smax v0.2s, v0.2s, v1.2s +; CHECK-NEXT: smax v0.2s, v0.2s, v2.2s ; CHECK-NEXT: ret entry: %conv = fptosi <2 x double> %x to <2 x i32> diff --git a/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll b/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll index 92fd3183393e..c45885a38f15 100644 --- a/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll +++ b/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll @@ -697,9 +697,9 @@ define <2 x i1> @test_signed_v2f32_v2i1(<2 x float> %f) { ; CHECK: // %bb.0: ; CHECK-NEXT: movi v1.2d, #0000000000000000 ; CHECK-NEXT: fcvtzs v0.2s, v0.2s +; CHECK-NEXT: movi v2.2d, #0xffffffffffffffff ; CHECK-NEXT: smin v0.2s, v0.2s, v1.2s -; CHECK-NEXT: movi v1.2d, #0xffffffffffffffff -; CHECK-NEXT: smax v0.2s, v0.2s, v1.2s +; CHECK-NEXT: smax v0.2s, v0.2s, v2.2s ; CHECK-NEXT: ret %x = call <2 x i1> @llvm.fptosi.sat.v2f32.v2i1(<2 x float> %f) ret <2 x i1> %x @@ -1620,9 +1620,9 @@ define <4 x i1> @test_signed_v4f16_v4i1(<4 x half> %f) { ; CHECK-FP16: // %bb.0: ; CHECK-FP16-NEXT: movi v1.2d, #0000000000000000 ; CHECK-FP16-NEXT: fcvtzs v0.4h, v0.4h +; CHECK-FP16-NEXT: movi v2.2d, #0xffffffffffffffff ; CHECK-FP16-NEXT: smin v0.4h, v0.4h, v1.4h -; CHECK-FP16-NEXT: movi v1.2d, #0xffffffffffffffff -; CHECK-FP16-NEXT: smax v0.4h, v0.4h, v1.4h +; CHECK-FP16-NEXT: smax v0.4h, v0.4h, v2.4h ; CHECK-FP16-NEXT: ret %x = call <4 x i1> @llvm.fptosi.sat.v4f16.v4i1(<4 x half> %f) ret <4 x i1> %x @@ -1668,9 +1668,9 @@ define <4 x i13> @test_signed_v4f16_v4i13(<4 x half> %f) { ; CHECK-FP16: // %bb.0: ; CHECK-FP16-NEXT: fcvtzs v0.4h, v0.4h ; CHECK-FP16-NEXT: mvni v1.4h, #240, lsl #8 +; CHECK-FP16-NEXT: movi v2.4h, #240, lsl #8 ; CHECK-FP16-NEXT: smin v0.4h, v0.4h, v1.4h -; CHECK-FP16-NEXT: movi v1.4h, #240, lsl #8 -; CHECK-FP16-NEXT: smax v0.4h, v0.4h, v1.4h +; CHECK-FP16-NEXT: smax v0.4h, v0.4h, v2.4h ; CHECK-FP16-NEXT: ret %x = call <4 x i13> @llvm.fptosi.sat.v4f16.v4i13(<4 x half> %f) ret <4 x i13> %x @@ -2103,9 +2103,9 @@ define <8 x i1> @test_signed_v8f16_v8i1(<8 x half> %f) { ; CHECK-FP16: // %bb.0: ; CHECK-FP16-NEXT: movi v1.2d, #0000000000000000 ; CHECK-FP16-NEXT: fcvtzs v0.8h, v0.8h +; CHECK-FP16-NEXT: movi v2.2d, #0xffffffffffffffff ; CHECK-FP16-NEXT: smin v0.8h, v0.8h, v1.8h -; CHECK-FP16-NEXT: movi v1.2d, #0xffffffffffffffff -; CHECK-FP16-NEXT: smax v0.8h, v0.8h, v1.8h +; CHECK-FP16-NEXT: smax v0.8h, v0.8h, v2.8h ; CHECK-FP16-NEXT: xtn v0.8b, v0.8h ; CHECK-FP16-NEXT: ret %x = call <8 x i1> @llvm.fptosi.sat.v8f16.v8i1(<8 x half> %f) @@ -2254,9 +2254,9 @@ define <8 x i13> @test_signed_v8f16_v8i13(<8 x half> %f) { ; CHECK-FP16: // %bb.0: ; CHECK-FP16-NEXT: fcvtzs v0.8h, v0.8h ; CHECK-FP16-NEXT: mvni v1.8h, #240, lsl #8 +; CHECK-FP16-NEXT: movi v2.8h, #240, lsl #8 ; CHECK-FP16-NEXT: smin v0.8h, v0.8h, v1.8h -; CHECK-FP16-NEXT: movi v1.8h, #240, lsl #8 -; CHECK-FP16-NEXT: smax v0.8h, v0.8h, v1.8h +; CHECK-FP16-NEXT: smax v0.8h, v0.8h, v2.8h ; CHECK-FP16-NEXT: ret %x = call <8 x i13> @llvm.fptosi.sat.v8f16.v8i13(<8 x half> %f) ret <8 x i13> %x diff --git a/llvm/test/CodeGen/AArch64/funnel-shift-rot.ll b/llvm/test/CodeGen/AArch64/funnel-shift-rot.ll index 181f2185893e..d39c09524e1a 100644 --- a/llvm/test/CodeGen/AArch64/funnel-shift-rot.ll +++ b/llvm/test/CodeGen/AArch64/funnel-shift-rot.ll @@ -78,9 +78,9 @@ define <4 x i32> @rotl_v4i32(<4 x i32> %x, <4 x i32> %z) { ; CHECK: // %bb.0: ; CHECK-NEXT: movi v2.4s, #31 ; CHECK-NEXT: neg v3.4s, v1.4s +; CHECK-NEXT: and v3.16b, v3.16b, v2.16b ; CHECK-NEXT: and v1.16b, v1.16b, v2.16b -; CHECK-NEXT: and v2.16b, v3.16b, v2.16b -; CHECK-NEXT: neg v2.4s, v2.4s +; CHECK-NEXT: neg v2.4s, v3.4s ; CHECK-NEXT: ushl v1.4s, v0.4s, v1.4s ; CHECK-NEXT: ushl v0.4s, v0.4s, v2.4s ; CHECK-NEXT: orr v0.16b, v1.16b, v0.16b diff --git a/llvm/test/CodeGen/AArch64/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll b/llvm/test/CodeGen/AArch64/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll index 97511639ec8c..cb9f04a7fac4 100644 --- a/llvm/test/CodeGen/AArch64/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll +++ b/llvm/test/CodeGen/AArch64/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll @@ -202,8 +202,8 @@ define <4 x i1> @vec_4xi32_splat_eq(<4 x i32> %x, <4 x i32> %y) nounwind { define <4 x i1> @vec_4xi32_nonsplat_eq(<4 x i32> %x, <4 x i32> %y) nounwind { ; CHECK-LABEL: vec_4xi32_nonsplat_eq: ; CHECK: // %bb.0: -; CHECK-NEXT: neg v1.4s, v1.4s ; CHECK-NEXT: adrp x8, .LCPI13_0 +; CHECK-NEXT: neg v1.4s, v1.4s ; CHECK-NEXT: ldr q2, [x8, :lo12:.LCPI13_0] ; CHECK-NEXT: ushl v1.4s, v2.4s, v1.4s ; CHECK-NEXT: and v0.16b, v1.16b, v0.16b diff --git a/llvm/test/CodeGen/AArch64/icmp.ll b/llvm/test/CodeGen/AArch64/icmp.ll index e7352fe03d01..8e10847e7aae 100644 --- a/llvm/test/CodeGen/AArch64/icmp.ll +++ b/llvm/test/CodeGen/AArch64/icmp.ll @@ -179,10 +179,10 @@ define <3 x i32> @v3i32_i32(<3 x i32> %a, <3 x i32> %b, <3 x i32> %d, <3 x i32> ; CHECK-GI-NEXT: mov w8, #-1 // =0xffffffff ; CHECK-GI-NEXT: fmov s1, w8 ; CHECK-GI-NEXT: mov v1.s[1], w8 -; CHECK-GI-NEXT: neg v5.4s, v4.4s ; CHECK-GI-NEXT: ushl v0.4s, v0.4s, v4.4s +; CHECK-GI-NEXT: neg v4.4s, v4.4s +; CHECK-GI-NEXT: sshl v0.4s, v0.4s, v4.4s ; CHECK-GI-NEXT: mov v1.s[2], w8 -; CHECK-GI-NEXT: sshl v0.4s, v0.4s, v5.4s ; CHECK-GI-NEXT: eor v1.16b, v0.16b, v1.16b ; CHECK-GI-NEXT: and v0.16b, v2.16b, v0.16b ; CHECK-GI-NEXT: and v1.16b, v3.16b, v1.16b diff --git a/llvm/test/CodeGen/AArch64/insert-extend.ll b/llvm/test/CodeGen/AArch64/insert-extend.ll index e4d2b516b8fb..0b730f6e7715 100644 --- a/llvm/test/CodeGen/AArch64/insert-extend.ll +++ b/llvm/test/CodeGen/AArch64/insert-extend.ll @@ -64,104 +64,104 @@ define i32 @large(ptr nocapture noundef readonly %p1, i32 noundef %st1, ptr noca ; CHECK-NEXT: ldr d3, [x11] ; CHECK-NEXT: ldr d4, [x10, x8] ; CHECK-NEXT: ldr d5, [x11, x9] +; CHECK-NEXT: shll2 v6.4s, v0.8h, #16 ; CHECK-NEXT: usubl v2.8h, v2.8b, v3.8b ; CHECK-NEXT: usubl v3.8h, v4.8b, v5.8b -; CHECK-NEXT: shll2 v4.4s, v0.8h, #16 -; CHECK-NEXT: shll2 v5.4s, v1.8h, #16 -; CHECK-NEXT: saddw v0.4s, v4.4s, v0.4h +; CHECK-NEXT: shll2 v4.4s, v1.8h, #16 +; CHECK-NEXT: saddw v0.4s, v6.4s, v0.4h ; CHECK-NEXT: shll2 v6.4s, v2.8h, #16 -; CHECK-NEXT: saddw v1.4s, v5.4s, v1.4h -; CHECK-NEXT: shll2 v4.4s, v3.8h, #16 -; CHECK-NEXT: saddw v2.4s, v6.4s, v2.4h -; CHECK-NEXT: saddw v3.4s, v4.4s, v3.4h +; CHECK-NEXT: shll2 v5.4s, v3.8h, #16 +; CHECK-NEXT: saddw v1.4s, v4.4s, v1.4h ; CHECK-NEXT: rev64 v4.4s, v0.4s +; CHECK-NEXT: saddw v2.4s, v6.4s, v2.4h +; CHECK-NEXT: saddw v3.4s, v5.4s, v3.4h ; CHECK-NEXT: rev64 v5.4s, v1.4s ; CHECK-NEXT: rev64 v6.4s, v2.4s -; CHECK-NEXT: rev64 v7.4s, v3.4s ; CHECK-NEXT: sub v4.4s, v0.4s, v4.4s ; CHECK-NEXT: addp v0.4s, v1.4s, v0.4s +; CHECK-NEXT: rev64 v7.4s, v3.4s ; CHECK-NEXT: sub v5.4s, v1.4s, v5.4s ; CHECK-NEXT: sub v6.4s, v2.4s, v6.4s ; CHECK-NEXT: addp v2.4s, v3.4s, v2.4s -; CHECK-NEXT: sub v7.4s, v3.4s, v7.4s ; CHECK-NEXT: zip1 v16.4s, v5.4s, v4.4s -; CHECK-NEXT: ext v1.16b, v2.16b, v2.16b, #8 +; CHECK-NEXT: sub v7.4s, v3.4s, v7.4s ; CHECK-NEXT: zip2 v3.4s, v6.4s, v7.4s ; CHECK-NEXT: mov v6.s[1], v7.s[0] ; CHECK-NEXT: ext v7.16b, v5.16b, v16.16b, #8 ; CHECK-NEXT: mov v5.s[3], v4.s[2] -; CHECK-NEXT: uzp1 v4.4s, v1.4s, v0.4s -; CHECK-NEXT: uzp2 v1.4s, v1.4s, v0.4s +; CHECK-NEXT: ext v4.16b, v2.16b, v2.16b, #8 ; CHECK-NEXT: mov v6.d[1], v7.d[1] ; CHECK-NEXT: mov v3.d[1], v5.d[1] +; CHECK-NEXT: uzp1 v1.4s, v4.4s, v0.4s +; CHECK-NEXT: uzp2 v4.4s, v4.4s, v0.4s ; CHECK-NEXT: uzp2 v5.4s, v2.4s, v0.4s ; CHECK-NEXT: uzp1 v0.4s, v2.4s, v0.4s -; CHECK-NEXT: sub v1.4s, v4.4s, v1.4s ; CHECK-NEXT: add v2.4s, v3.4s, v6.4s ; CHECK-NEXT: sub v3.4s, v6.4s, v3.4s +; CHECK-NEXT: sub v1.4s, v1.4s, v4.4s ; CHECK-NEXT: add v0.4s, v5.4s, v0.4s -; CHECK-NEXT: rev64 v6.4s, v1.4s ; CHECK-NEXT: rev64 v4.4s, v2.4s ; CHECK-NEXT: rev64 v5.4s, v3.4s -; CHECK-NEXT: addp v16.4s, v1.4s, v3.4s +; CHECK-NEXT: rev64 v6.4s, v1.4s ; CHECK-NEXT: rev64 v7.4s, v0.4s +; CHECK-NEXT: addp v16.4s, v1.4s, v3.4s ; CHECK-NEXT: addp v17.4s, v0.4s, v2.4s -; CHECK-NEXT: sub v1.4s, v1.4s, v6.4s ; CHECK-NEXT: sub v3.4s, v3.4s, v5.4s ; CHECK-NEXT: sub v2.4s, v2.4s, v4.4s +; CHECK-NEXT: sub v1.4s, v1.4s, v6.4s ; CHECK-NEXT: sub v0.4s, v0.4s, v7.4s ; CHECK-NEXT: zip1 v18.4s, v17.4s, v17.4s -; CHECK-NEXT: ext v6.16b, v1.16b, v16.16b, #8 ; CHECK-NEXT: ext v4.16b, v17.16b, v2.16b, #4 ; CHECK-NEXT: ext v5.16b, v16.16b, v3.16b, #4 ; CHECK-NEXT: mov v20.16b, v3.16b +; CHECK-NEXT: ext v6.16b, v1.16b, v16.16b, #8 ; CHECK-NEXT: ext v7.16b, v0.16b, v17.16b, #4 ; CHECK-NEXT: mov v21.16b, v2.16b ; CHECK-NEXT: trn2 v0.4s, v18.4s, v0.4s -; CHECK-NEXT: ext v19.16b, v6.16b, v1.16b, #4 -; CHECK-NEXT: mov v1.s[2], v16.s[1] ; CHECK-NEXT: mov v20.s[2], v16.s[3] ; CHECK-NEXT: zip2 v4.4s, v4.4s, v17.4s ; CHECK-NEXT: zip2 v5.4s, v5.4s, v16.4s ; CHECK-NEXT: mov v21.s[2], v17.s[3] +; CHECK-NEXT: ext v19.16b, v6.16b, v1.16b, #4 ; CHECK-NEXT: ext v7.16b, v7.16b, v7.16b, #4 -; CHECK-NEXT: mov v18.16b, v1.16b +; CHECK-NEXT: mov v1.s[2], v16.s[1] ; CHECK-NEXT: ext v2.16b, v2.16b, v4.16b, #12 ; CHECK-NEXT: ext v3.16b, v3.16b, v5.16b, #12 ; CHECK-NEXT: uzp2 v4.4s, v6.4s, v19.4s ; CHECK-NEXT: mov v5.16b, v7.16b ; CHECK-NEXT: mov v6.16b, v20.16b +; CHECK-NEXT: mov v18.16b, v1.16b ; CHECK-NEXT: mov v19.16b, v21.16b -; CHECK-NEXT: mov v18.s[1], v16.s[0] ; CHECK-NEXT: sub v7.4s, v0.4s, v7.4s ; CHECK-NEXT: mov v6.s[1], v16.s[2] ; CHECK-NEXT: mov v5.s[0], v17.s[1] +; CHECK-NEXT: mov v18.s[1], v16.s[0] ; CHECK-NEXT: mov v19.s[1], v17.s[2] ; CHECK-NEXT: sub v1.4s, v1.4s, v4.4s ; CHECK-NEXT: sub v16.4s, v20.4s, v3.4s ; CHECK-NEXT: sub v17.4s, v21.4s, v2.4s -; CHECK-NEXT: add v4.4s, v18.4s, v4.4s ; CHECK-NEXT: add v3.4s, v6.4s, v3.4s ; CHECK-NEXT: add v0.4s, v0.4s, v5.4s +; CHECK-NEXT: add v4.4s, v18.4s, v4.4s ; CHECK-NEXT: add v2.4s, v19.4s, v2.4s -; CHECK-NEXT: mov v4.d[1], v1.d[1] ; CHECK-NEXT: mov v3.d[1], v16.d[1] ; CHECK-NEXT: mov v0.d[1], v7.d[1] +; CHECK-NEXT: mov v4.d[1], v1.d[1] ; CHECK-NEXT: mov v2.d[1], v17.d[1] -; CHECK-NEXT: cmlt v6.8h, v4.8h, #0 ; CHECK-NEXT: cmlt v1.8h, v3.8h, #0 ; CHECK-NEXT: cmlt v5.8h, v0.8h, #0 +; CHECK-NEXT: cmlt v6.8h, v4.8h, #0 ; CHECK-NEXT: cmlt v7.8h, v2.8h, #0 -; CHECK-NEXT: add v4.4s, v6.4s, v4.4s ; CHECK-NEXT: add v3.4s, v1.4s, v3.4s ; CHECK-NEXT: add v0.4s, v5.4s, v0.4s +; CHECK-NEXT: add v4.4s, v6.4s, v4.4s ; CHECK-NEXT: add v2.4s, v7.4s, v2.4s ; CHECK-NEXT: eor v1.16b, v3.16b, v1.16b +; CHECK-NEXT: eor v0.16b, v0.16b, v5.16b ; CHECK-NEXT: eor v2.16b, v2.16b, v7.16b ; CHECK-NEXT: eor v3.16b, v4.16b, v6.16b -; CHECK-NEXT: eor v0.16b, v0.16b, v5.16b -; CHECK-NEXT: add v1.4s, v2.4s, v1.4s ; CHECK-NEXT: add v0.4s, v0.4s, v3.4s +; CHECK-NEXT: add v1.4s, v2.4s, v1.4s ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: addv s0, v0.4s ; CHECK-NEXT: fmov w8, s0 diff --git a/llvm/test/CodeGen/AArch64/insert-subvector-res-legalization.ll b/llvm/test/CodeGen/AArch64/insert-subvector-res-legalization.ll index 29f9c0336bbc..542b2e90ffc1 100644 --- a/llvm/test/CodeGen/AArch64/insert-subvector-res-legalization.ll +++ b/llvm/test/CodeGen/AArch64/insert-subvector-res-legalization.ll @@ -9,9 +9,9 @@ define @vec_scalable_subvec_scalable_idx_zero_i8(ptr %a, ptr % ; CHECK-LABEL: vec_scalable_subvec_scalable_idx_zero_i8: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.h -; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: ld1b { z0.h }, p0/z, [x0] -; CHECK-NEXT: ld1b { z1.s }, p1/z, [x1] +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: ld1b { z1.s }, p0/z, [x1] ; CHECK-NEXT: uunpkhi z0.s, z0.h ; CHECK-NEXT: uzp1 z0.h, z1.h, z0.h ; CHECK-NEXT: ret @@ -25,9 +25,9 @@ define @vec_scalable_subvec_scalable_idx_nonzero_i8(ptr %a, pt ; CHECK-LABEL: vec_scalable_subvec_scalable_idx_nonzero_i8: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.h -; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: ld1b { z0.h }, p0/z, [x0] -; CHECK-NEXT: ld1b { z1.s }, p1/z, [x1] +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: ld1b { z1.s }, p0/z, [x1] ; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h ; CHECK-NEXT: ret @@ -41,9 +41,9 @@ define @vec_scalable_subvec_scalable_idx_zero_i16(ptr %a, ptr ; CHECK-LABEL: vec_scalable_subvec_scalable_idx_zero_i16: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: ld1h { z0.s }, p0/z, [x0] -; CHECK-NEXT: ld1h { z1.d }, p1/z, [x1] +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: ld1h { z1.d }, p0/z, [x1] ; CHECK-NEXT: uunpkhi z0.d, z0.s ; CHECK-NEXT: uzp1 z0.s, z1.s, z0.s ; CHECK-NEXT: ret @@ -57,9 +57,9 @@ define @vec_scalable_subvec_scalable_idx_nonzero_i16(ptr %a, ; CHECK-LABEL: vec_scalable_subvec_scalable_idx_nonzero_i16: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: ld1h { z0.s }, p0/z, [x0] -; CHECK-NEXT: ld1h { z1.d }, p1/z, [x1] +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: ld1h { z1.d }, p0/z, [x1] ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: uzp1 z0.s, z0.s, z1.s ; CHECK-NEXT: ret @@ -76,10 +76,10 @@ define @vec_scalable_subvec_fixed_idx_zero_i8(ptr %a, ptr %b) ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: ldr d0, [x1] -; CHECK-NEXT: ptrue p1.h, vl8 -; CHECK-NEXT: ushll v0.8h, v0.8b, #0 ; CHECK-NEXT: ld1b { z1.h }, p0/z, [x0] -; CHECK-NEXT: sel z0.h, p1, z0.h, z1.h +; CHECK-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-NEXT: ptrue p0.h, vl8 +; CHECK-NEXT: sel z0.h, p0, z0.h, z1.h ; CHECK-NEXT: ret %vec = load , ptr %a %subvec = load <8 x i8>, ptr %b @@ -92,19 +92,19 @@ define @vec_scalable_subvec_fixed_idx_nonzero_i8(ptr %a, ptr % ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: cnth x8 -; CHECK-NEXT: ldr d0, [x1] +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: ldr d1, [x1] ; CHECK-NEXT: sub x8, x8, #8 ; CHECK-NEXT: mov w9, #8 // =0x8 ; CHECK-NEXT: cmp x8, #8 -; CHECK-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-NEXT: ld1b { z0.h }, p0/z, [x0] +; CHECK-NEXT: ushll v1.8h, v1.8b, #0 ; CHECK-NEXT: csel x8, x8, x9, lo ; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: lsl x8, x8, #1 -; CHECK-NEXT: ld1b { z1.h }, p0/z, [x0] -; CHECK-NEXT: st1h { z1.h }, p0, [sp] -; CHECK-NEXT: str q0, [x9, x8] +; CHECK-NEXT: st1h { z0.h }, p0, [sp] +; CHECK-NEXT: str q1, [x9, x8] ; CHECK-NEXT: ld1h { z0.h }, p0/z, [sp] ; CHECK-NEXT: addvl sp, sp, #1 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload @@ -120,10 +120,10 @@ define @vec_scalable_subvec_fixed_idx_zero_i16(ptr %a, ptr %b ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldr d0, [x1] -; CHECK-NEXT: ptrue p1.s, vl4 -; CHECK-NEXT: ushll v0.4s, v0.4h, #0 ; CHECK-NEXT: ld1h { z1.s }, p0/z, [x0] -; CHECK-NEXT: sel z0.s, p1, z0.s, z1.s +; CHECK-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-NEXT: ptrue p0.s, vl4 +; CHECK-NEXT: sel z0.s, p0, z0.s, z1.s ; CHECK-NEXT: ret %vec = load , ptr %a %subvec = load <4 x i16>, ptr %b @@ -136,19 +136,19 @@ define @vec_scalable_subvec_fixed_idx_nonzero_i16(ptr %a, ptr ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: cntw x8 -; CHECK-NEXT: ldr d0, [x1] +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: ldr d1, [x1] ; CHECK-NEXT: sub x8, x8, #4 ; CHECK-NEXT: mov w9, #4 // =0x4 ; CHECK-NEXT: cmp x8, #4 -; CHECK-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-NEXT: ld1h { z0.s }, p0/z, [x0] +; CHECK-NEXT: ushll v1.4s, v1.4h, #0 ; CHECK-NEXT: csel x8, x8, x9, lo ; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: lsl x8, x8, #2 -; CHECK-NEXT: ld1h { z1.s }, p0/z, [x0] -; CHECK-NEXT: st1w { z1.s }, p0, [sp] -; CHECK-NEXT: str q0, [x9, x8] +; CHECK-NEXT: st1w { z0.s }, p0, [sp] +; CHECK-NEXT: str q1, [x9, x8] ; CHECK-NEXT: ld1w { z0.s }, p0/z, [sp] ; CHECK-NEXT: addvl sp, sp, #1 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload @@ -164,10 +164,10 @@ define @vec_scalable_subvec_fixed_idx_zero_i32(ptr %a, ptr %b ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldr d0, [x1] -; CHECK-NEXT: ptrue p1.d, vl2 -; CHECK-NEXT: ushll v0.2d, v0.2s, #0 ; CHECK-NEXT: ld1w { z1.d }, p0/z, [x0] -; CHECK-NEXT: sel z0.d, p1, z0.d, z1.d +; CHECK-NEXT: ushll v0.2d, v0.2s, #0 +; CHECK-NEXT: ptrue p0.d, vl2 +; CHECK-NEXT: sel z0.d, p0, z0.d, z1.d ; CHECK-NEXT: ret %vec = load , ptr %a %subvec = load <2 x i32>, ptr %b @@ -180,19 +180,19 @@ define @vec_scalable_subvec_fixed_idx_nonzero_i32(ptr %a, ptr ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: cntd x8 -; CHECK-NEXT: ldr d0, [x1] +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: ldr d1, [x1] ; CHECK-NEXT: sub x8, x8, #2 ; CHECK-NEXT: mov w9, #2 // =0x2 ; CHECK-NEXT: cmp x8, #2 -; CHECK-NEXT: ushll v0.2d, v0.2s, #0 +; CHECK-NEXT: ld1w { z0.d }, p0/z, [x0] +; CHECK-NEXT: ushll v1.2d, v1.2s, #0 ; CHECK-NEXT: csel x8, x8, x9, lo ; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: lsl x8, x8, #3 -; CHECK-NEXT: ld1w { z1.d }, p0/z, [x0] -; CHECK-NEXT: st1d { z1.d }, p0, [sp] -; CHECK-NEXT: str q0, [x9, x8] +; CHECK-NEXT: st1d { z0.d }, p0, [sp] +; CHECK-NEXT: str q1, [x9, x8] ; CHECK-NEXT: ld1d { z0.d }, p0/z, [sp] ; CHECK-NEXT: addvl sp, sp, #1 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload diff --git a/llvm/test/CodeGen/AArch64/intrinsic-cttz-elts-sve.ll b/llvm/test/CodeGen/AArch64/intrinsic-cttz-elts-sve.ll index 1a4ab6ab334a..9bd2ed240810 100644 --- a/llvm/test/CodeGen/AArch64/intrinsic-cttz-elts-sve.ll +++ b/llvm/test/CodeGen/AArch64/intrinsic-cttz-elts-sve.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme < %s | FileCheck %s @@ -31,28 +32,28 @@ define i32 @ctz_nxv32i1( %a) #0 { ; CHECK-NEXT: neg x8, x8 ; CHECK-NEXT: punpklo p3.h, p1.b ; CHECK-NEXT: rdvl x9, #2 -; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: rdvl x8, #-1 -; CHECK-NEXT: punpkhi p1.h, p1.b +; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: mov z2.h, w8 -; CHECK-NEXT: inch z0.h, all, mul #4 +; CHECK-NEXT: punpkhi p1.h, p1.b ; CHECK-NEXT: mov z3.h, p2/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: ptrue p2.h +; CHECK-NEXT: inch z0.h, all, mul #4 ; CHECK-NEXT: mov z5.h, p3/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: add z1.h, z0.h, z1.h -; CHECK-NEXT: add z4.h, z0.h, z2.h ; CHECK-NEXT: mov z6.h, p0/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z7.h, p1/z, #-1 // =0xffffffffffffffff +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: add z1.h, z0.h, z1.h +; CHECK-NEXT: add z4.h, z0.h, z2.h ; CHECK-NEXT: and z0.d, z0.d, z3.d ; CHECK-NEXT: add z2.h, z1.h, z2.h ; CHECK-NEXT: and z3.d, z4.d, z5.d ; CHECK-NEXT: and z1.d, z1.d, z6.d ; CHECK-NEXT: and z2.d, z2.d, z7.d -; CHECK-NEXT: umax z0.h, p2/m, z0.h, z3.h -; CHECK-NEXT: umax z1.h, p2/m, z1.h, z2.h -; CHECK-NEXT: umax z0.h, p2/m, z0.h, z1.h -; CHECK-NEXT: umaxv h0, p2, z0.h +; CHECK-NEXT: umax z0.h, p0/m, z0.h, z3.h +; CHECK-NEXT: umax z1.h, p0/m, z1.h, z2.h +; CHECK-NEXT: umax z0.h, p0/m, z0.h, z1.h +; CHECK-NEXT: umaxv h0, p0, z0.h ; CHECK-NEXT: fmov w8, s0 ; CHECK-NEXT: sub w8, w9, w8 ; CHECK-NEXT: and w0, w8, #0xffff @@ -65,12 +66,12 @@ define i32 @ctz_nxv4i32( %a) #0 { ; CHECK-LABEL: ctz_nxv4i32: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: index z1.s, #0, #-1 ; CHECK-NEXT: cntw x9 -; CHECK-NEXT: incw z1.s ; CHECK-NEXT: cmpne p1.s, p0/z, z0.s, #0 -; CHECK-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: and z0.d, z1.d, z0.d +; CHECK-NEXT: index z0.s, #0, #-1 +; CHECK-NEXT: mov z1.s, p1/z, #-1 // =0xffffffffffffffff +; CHECK-NEXT: incw z0.s +; CHECK-NEXT: and z0.d, z0.d, z1.d ; CHECK-NEXT: and z0.s, z0.s, #0xff ; CHECK-NEXT: umaxv s0, p0, z0.s ; CHECK-NEXT: fmov w8, s0 @@ -87,38 +88,38 @@ define i64 @vscale_4096( %a) #1 { ; CHECK-LABEL: vscale_4096: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: index z1.s, #0, #-1 ; CHECK-NEXT: cntw x8 -; CHECK-NEXT: cnth x9 ; CHECK-NEXT: neg x8, x8 -; CHECK-NEXT: mov z1.s, w8 -; CHECK-NEXT: neg x8, x9 ; CHECK-NEXT: rdvl x9, #1 -; CHECK-NEXT: mov z2.s, w8 ; CHECK-NEXT: cmpne p0.b, p0/z, z0.b, #0 -; CHECK-NEXT: index z0.s, #0, #-1 +; CHECK-NEXT: mov z0.s, w8 +; CHECK-NEXT: cnth x8 +; CHECK-NEXT: neg x8, x8 +; CHECK-NEXT: incw z1.s, all, mul #4 +; CHECK-NEXT: mov z2.s, w8 ; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: punpkhi p0.h, p0.b -; CHECK-NEXT: incw z0.s, all, mul #4 -; CHECK-NEXT: add z1.s, z0.s, z1.s -; CHECK-NEXT: add z5.s, z0.s, z2.s +; CHECK-NEXT: add z0.s, z1.s, z0.s +; CHECK-NEXT: add z4.s, z1.s, z2.s ; CHECK-NEXT: punpkhi p2.h, p1.b ; CHECK-NEXT: punpkhi p3.h, p0.b ; CHECK-NEXT: punpklo p0.h, p0.b -; CHECK-NEXT: add z2.s, z1.s, z2.s -; CHECK-NEXT: punpklo p1.h, p1.b ; CHECK-NEXT: mov z3.s, p2/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: ptrue p2.s -; CHECK-NEXT: mov z4.s, p3/z, #-1 // =0xffffffffffffffff +; CHECK-NEXT: add z2.s, z0.s, z2.s +; CHECK-NEXT: punpklo p1.h, p1.b +; CHECK-NEXT: mov z5.s, p3/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z6.s, p0/z, #-1 // =0xffffffffffffffff +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z7.s, p1/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: and z1.d, z1.d, z3.d -; CHECK-NEXT: and z2.d, z2.d, z4.d -; CHECK-NEXT: and z3.d, z5.d, z6.d -; CHECK-NEXT: and z0.d, z0.d, z7.d -; CHECK-NEXT: umax z1.s, p2/m, z1.s, z2.s -; CHECK-NEXT: umax z0.s, p2/m, z0.s, z3.s -; CHECK-NEXT: umax z0.s, p2/m, z0.s, z1.s -; CHECK-NEXT: umaxv s0, p2, z0.s +; CHECK-NEXT: and z0.d, z0.d, z3.d +; CHECK-NEXT: and z2.d, z2.d, z5.d +; CHECK-NEXT: and z3.d, z4.d, z6.d +; CHECK-NEXT: and z1.d, z1.d, z7.d +; CHECK-NEXT: umax z0.s, p0/m, z0.s, z2.s +; CHECK-NEXT: umax z1.s, p0/m, z1.s, z3.s +; CHECK-NEXT: umax z0.s, p0/m, z0.s, z1.s +; CHECK-NEXT: umaxv s0, p0, z0.s ; CHECK-NEXT: fmov w8, s0 ; CHECK-NEXT: sub w0, w9, w8 ; CHECK-NEXT: ret @@ -130,21 +131,21 @@ define i64 @vscale_4096_poison( %a) #1 { ; CHECK-LABEL: vscale_4096_poison: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: index z1.h, #0, #-1 ; CHECK-NEXT: cnth x8 -; CHECK-NEXT: rdvl x9, #1 ; CHECK-NEXT: neg x8, x8 -; CHECK-NEXT: mov z1.h, w8 +; CHECK-NEXT: rdvl x9, #1 ; CHECK-NEXT: cmpne p0.b, p0/z, z0.b, #0 -; CHECK-NEXT: index z0.h, #0, #-1 +; CHECK-NEXT: mov z0.h, w8 +; CHECK-NEXT: inch z1.h, all, mul #2 ; CHECK-NEXT: punpkhi p1.h, p0.b ; CHECK-NEXT: punpklo p0.h, p0.b -; CHECK-NEXT: inch z0.h, all, mul #2 -; CHECK-NEXT: add z1.h, z0.h, z1.h +; CHECK-NEXT: add z0.h, z1.h, z0.h ; CHECK-NEXT: mov z2.h, p1/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z3.h, p0/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: ptrue p0.h -; CHECK-NEXT: and z1.d, z1.d, z2.d -; CHECK-NEXT: and z0.d, z0.d, z3.d +; CHECK-NEXT: and z0.d, z0.d, z2.d +; CHECK-NEXT: and z1.d, z1.d, z3.d ; CHECK-NEXT: umax z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: umaxv h0, p0, z0.h ; CHECK-NEXT: fmov w8, s0 @@ -161,16 +162,16 @@ define i32 @ctz_nxv8i1_no_range( %a) { ; CHECK-LABEL: ctz_nxv8i1_no_range: ; CHECK: // %bb.0: ; CHECK-NEXT: index z0.s, #0, #-1 -; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: cntw x8 -; CHECK-NEXT: punpkhi p0.h, p0.b +; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: neg x8, x8 +; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: cnth x9 ; CHECK-NEXT: mov z1.s, w8 -; CHECK-NEXT: incw z0.s, all, mul #2 ; CHECK-NEXT: mov z2.s, p1/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z3.s, p0/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: incw z0.s, all, mul #2 ; CHECK-NEXT: add z1.s, z0.s, z1.s ; CHECK-NEXT: and z0.d, z0.d, z2.d ; CHECK-NEXT: and z1.d, z1.d, z3.d @@ -212,8 +213,8 @@ define i32 @ctz_nxv16i1_poison( %pg, %a) { define i32 @ctz_and_nxv16i1( %pg, %a, %b) { ; CHECK-LABEL: ctz_and_nxv16i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.b ; CHECK-NEXT: cmpne p0.b, p0/z, z0.b, z1.b +; CHECK-NEXT: ptrue p1.b ; CHECK-NEXT: brkb p0.b, p1/z, p0.b ; CHECK-NEXT: cntp x0, p0, p0.b ; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0 diff --git a/llvm/test/CodeGen/AArch64/itofp.ll b/llvm/test/CodeGen/AArch64/itofp.ll index f5a7b5dc9f49..ae4ced258bb8 100644 --- a/llvm/test/CodeGen/AArch64/itofp.ll +++ b/llvm/test/CodeGen/AArch64/itofp.ll @@ -6555,18 +6555,18 @@ define <3 x bfloat> @stofp_v3i64_v3bf16(<3 x i64> %a) { ; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2 ; CHECK-NEXT: mov v0.d[1], v1.d[0] ; CHECK-NEXT: scvtf v1.2d, v2.2d -; CHECK-NEXT: movi v2.4s, #1 +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: scvtf v0.2d, v0.2d ; CHECK-NEXT: fcvtn v0.2s, v0.2d ; CHECK-NEXT: fcvtn2 v0.4s, v1.2d -; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: movi v1.4s, #1 ; CHECK-NEXT: ushr v3.4s, v0.4s, #16 -; CHECK-NEXT: add v1.4s, v0.4s, v1.4s -; CHECK-NEXT: and v2.16b, v3.16b, v2.16b -; CHECK-NEXT: add v1.4s, v2.4s, v1.4s -; CHECK-NEXT: fcmeq v2.4s, v0.4s, v0.4s +; CHECK-NEXT: add v2.4s, v0.4s, v2.4s +; CHECK-NEXT: and v1.16b, v3.16b, v1.16b +; CHECK-NEXT: fcmeq v3.4s, v0.4s, v0.4s ; CHECK-NEXT: orr v0.4s, #64, lsl #16 -; CHECK-NEXT: bit v0.16b, v1.16b, v2.16b +; CHECK-NEXT: add v1.4s, v1.4s, v2.4s +; CHECK-NEXT: bit v0.16b, v1.16b, v3.16b ; CHECK-NEXT: shrn v0.4h, v0.4s, #16 ; CHECK-NEXT: ret entry: @@ -6582,18 +6582,18 @@ define <3 x bfloat> @utofp_v3i64_v3bf16(<3 x i64> %a) { ; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2 ; CHECK-NEXT: mov v0.d[1], v1.d[0] ; CHECK-NEXT: ucvtf v1.2d, v2.2d -; CHECK-NEXT: movi v2.4s, #1 +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: ucvtf v0.2d, v0.2d ; CHECK-NEXT: fcvtn v0.2s, v0.2d ; CHECK-NEXT: fcvtn2 v0.4s, v1.2d -; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: movi v1.4s, #1 ; CHECK-NEXT: ushr v3.4s, v0.4s, #16 -; CHECK-NEXT: add v1.4s, v0.4s, v1.4s -; CHECK-NEXT: and v2.16b, v3.16b, v2.16b -; CHECK-NEXT: add v1.4s, v2.4s, v1.4s -; CHECK-NEXT: fcmeq v2.4s, v0.4s, v0.4s +; CHECK-NEXT: add v2.4s, v0.4s, v2.4s +; CHECK-NEXT: and v1.16b, v3.16b, v1.16b +; CHECK-NEXT: fcmeq v3.4s, v0.4s, v0.4s ; CHECK-NEXT: orr v0.4s, #64, lsl #16 -; CHECK-NEXT: bit v0.16b, v1.16b, v2.16b +; CHECK-NEXT: add v1.4s, v1.4s, v2.4s +; CHECK-NEXT: bit v0.16b, v1.16b, v3.16b ; CHECK-NEXT: shrn v0.4h, v0.4s, #16 ; CHECK-NEXT: ret entry: @@ -6606,17 +6606,17 @@ define <4 x bfloat> @stofp_v4i64_v4bf16(<4 x i64> %a) { ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: scvtf v0.2d, v0.2d ; CHECK-NEXT: scvtf v1.2d, v1.2d -; CHECK-NEXT: movi v2.4s, #1 +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: fcvtn v0.2s, v0.2d ; CHECK-NEXT: fcvtn2 v0.4s, v1.2d -; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: movi v1.4s, #1 ; CHECK-NEXT: ushr v3.4s, v0.4s, #16 -; CHECK-NEXT: add v1.4s, v0.4s, v1.4s -; CHECK-NEXT: and v2.16b, v3.16b, v2.16b -; CHECK-NEXT: add v1.4s, v2.4s, v1.4s -; CHECK-NEXT: fcmeq v2.4s, v0.4s, v0.4s +; CHECK-NEXT: add v2.4s, v0.4s, v2.4s +; CHECK-NEXT: and v1.16b, v3.16b, v1.16b +; CHECK-NEXT: fcmeq v3.4s, v0.4s, v0.4s ; CHECK-NEXT: orr v0.4s, #64, lsl #16 -; CHECK-NEXT: bit v0.16b, v1.16b, v2.16b +; CHECK-NEXT: add v1.4s, v1.4s, v2.4s +; CHECK-NEXT: bit v0.16b, v1.16b, v3.16b ; CHECK-NEXT: shrn v0.4h, v0.4s, #16 ; CHECK-NEXT: ret entry: @@ -6629,17 +6629,17 @@ define <4 x bfloat> @utofp_v4i64_v4bf16(<4 x i64> %a) { ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: ucvtf v0.2d, v0.2d ; CHECK-NEXT: ucvtf v1.2d, v1.2d -; CHECK-NEXT: movi v2.4s, #1 +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: fcvtn v0.2s, v0.2d ; CHECK-NEXT: fcvtn2 v0.4s, v1.2d -; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: movi v1.4s, #1 ; CHECK-NEXT: ushr v3.4s, v0.4s, #16 -; CHECK-NEXT: add v1.4s, v0.4s, v1.4s -; CHECK-NEXT: and v2.16b, v3.16b, v2.16b -; CHECK-NEXT: add v1.4s, v2.4s, v1.4s -; CHECK-NEXT: fcmeq v2.4s, v0.4s, v0.4s +; CHECK-NEXT: add v2.4s, v0.4s, v2.4s +; CHECK-NEXT: and v1.16b, v3.16b, v1.16b +; CHECK-NEXT: fcmeq v3.4s, v0.4s, v0.4s ; CHECK-NEXT: orr v0.4s, #64, lsl #16 -; CHECK-NEXT: bit v0.16b, v1.16b, v2.16b +; CHECK-NEXT: add v1.4s, v1.4s, v2.4s +; CHECK-NEXT: bit v0.16b, v1.16b, v3.16b ; CHECK-NEXT: shrn v0.4h, v0.4s, #16 ; CHECK-NEXT: ret entry: @@ -6658,22 +6658,22 @@ define <8 x bfloat> @stofp_v8i64_v8bf16(<8 x i64> %a) { ; CHECK-NEXT: fcvtn v0.2s, v0.2d ; CHECK-NEXT: fcvtn2 v2.4s, v3.2d ; CHECK-NEXT: fcvtn2 v0.4s, v1.2d -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: movi v3.4s, #1 +; CHECK-NEXT: movi v1.4s, #1 +; CHECK-NEXT: movi v3.4s, #127, msl #8 ; CHECK-NEXT: ushr v4.4s, v2.4s, #16 ; CHECK-NEXT: ushr v5.4s, v0.4s, #16 -; CHECK-NEXT: add v6.4s, v2.4s, v1.4s -; CHECK-NEXT: add v1.4s, v0.4s, v1.4s -; CHECK-NEXT: and v4.16b, v4.16b, v3.16b -; CHECK-NEXT: and v3.16b, v5.16b, v3.16b +; CHECK-NEXT: add v6.4s, v2.4s, v3.4s +; CHECK-NEXT: add v3.4s, v0.4s, v3.4s +; CHECK-NEXT: and v4.16b, v4.16b, v1.16b +; CHECK-NEXT: and v1.16b, v5.16b, v1.16b ; CHECK-NEXT: fcmeq v5.4s, v2.4s, v2.4s ; CHECK-NEXT: orr v2.4s, #64, lsl #16 ; CHECK-NEXT: add v4.4s, v4.4s, v6.4s -; CHECK-NEXT: add v1.4s, v3.4s, v1.4s -; CHECK-NEXT: fcmeq v3.4s, v0.4s, v0.4s +; CHECK-NEXT: fcmeq v6.4s, v0.4s, v0.4s +; CHECK-NEXT: add v1.4s, v1.4s, v3.4s ; CHECK-NEXT: orr v0.4s, #64, lsl #16 ; CHECK-NEXT: bit v2.16b, v4.16b, v5.16b -; CHECK-NEXT: bit v0.16b, v1.16b, v3.16b +; CHECK-NEXT: bit v0.16b, v1.16b, v6.16b ; CHECK-NEXT: uzp2 v0.8h, v0.8h, v2.8h ; CHECK-NEXT: ret entry: @@ -6692,22 +6692,22 @@ define <8 x bfloat> @utofp_v8i64_v8bf16(<8 x i64> %a) { ; CHECK-NEXT: fcvtn v0.2s, v0.2d ; CHECK-NEXT: fcvtn2 v2.4s, v3.2d ; CHECK-NEXT: fcvtn2 v0.4s, v1.2d -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: movi v3.4s, #1 +; CHECK-NEXT: movi v1.4s, #1 +; CHECK-NEXT: movi v3.4s, #127, msl #8 ; CHECK-NEXT: ushr v4.4s, v2.4s, #16 ; CHECK-NEXT: ushr v5.4s, v0.4s, #16 -; CHECK-NEXT: add v6.4s, v2.4s, v1.4s -; CHECK-NEXT: add v1.4s, v0.4s, v1.4s -; CHECK-NEXT: and v4.16b, v4.16b, v3.16b -; CHECK-NEXT: and v3.16b, v5.16b, v3.16b +; CHECK-NEXT: add v6.4s, v2.4s, v3.4s +; CHECK-NEXT: add v3.4s, v0.4s, v3.4s +; CHECK-NEXT: and v4.16b, v4.16b, v1.16b +; CHECK-NEXT: and v1.16b, v5.16b, v1.16b ; CHECK-NEXT: fcmeq v5.4s, v2.4s, v2.4s ; CHECK-NEXT: orr v2.4s, #64, lsl #16 ; CHECK-NEXT: add v4.4s, v4.4s, v6.4s -; CHECK-NEXT: add v1.4s, v3.4s, v1.4s -; CHECK-NEXT: fcmeq v3.4s, v0.4s, v0.4s +; CHECK-NEXT: fcmeq v6.4s, v0.4s, v0.4s +; CHECK-NEXT: add v1.4s, v1.4s, v3.4s ; CHECK-NEXT: orr v0.4s, #64, lsl #16 ; CHECK-NEXT: bit v2.16b, v4.16b, v5.16b -; CHECK-NEXT: bit v0.16b, v1.16b, v3.16b +; CHECK-NEXT: bit v0.16b, v1.16b, v6.16b ; CHECK-NEXT: uzp2 v0.8h, v0.8h, v2.8h ; CHECK-NEXT: ret entry: @@ -6718,51 +6718,51 @@ entry: define <16 x bfloat> @stofp_v16i64_v16bf16(<16 x i64> %a) { ; CHECK-LABEL: stofp_v16i64_v16bf16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: scvtf v2.2d, v2.2d ; CHECK-NEXT: scvtf v0.2d, v0.2d +; CHECK-NEXT: scvtf v2.2d, v2.2d ; CHECK-NEXT: scvtf v6.2d, v6.2d ; CHECK-NEXT: scvtf v4.2d, v4.2d -; CHECK-NEXT: scvtf v3.2d, v3.2d ; CHECK-NEXT: scvtf v1.2d, v1.2d +; CHECK-NEXT: scvtf v3.2d, v3.2d ; CHECK-NEXT: scvtf v7.2d, v7.2d ; CHECK-NEXT: scvtf v5.2d, v5.2d -; CHECK-NEXT: fcvtn v2.2s, v2.2d ; CHECK-NEXT: fcvtn v0.2s, v0.2d +; CHECK-NEXT: fcvtn v2.2s, v2.2d ; CHECK-NEXT: fcvtn v6.2s, v6.2d ; CHECK-NEXT: fcvtn v4.2s, v4.2d -; CHECK-NEXT: fcvtn2 v2.4s, v3.2d ; CHECK-NEXT: fcvtn2 v0.4s, v1.2d -; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: fcvtn2 v2.4s, v3.2d ; CHECK-NEXT: fcvtn2 v6.4s, v7.2d ; CHECK-NEXT: fcvtn2 v4.4s, v5.2d -; CHECK-NEXT: movi v3.4s, #1 -; CHECK-NEXT: ushr v5.4s, v2.4s, #16 +; CHECK-NEXT: movi v1.4s, #1 +; CHECK-NEXT: movi v3.4s, #127, msl #8 ; CHECK-NEXT: ushr v7.4s, v0.4s, #16 -; CHECK-NEXT: add v17.4s, v2.4s, v1.4s -; CHECK-NEXT: add v19.4s, v0.4s, v1.4s +; CHECK-NEXT: ushr v5.4s, v2.4s, #16 ; CHECK-NEXT: ushr v16.4s, v6.4s, #16 -; CHECK-NEXT: ushr v18.4s, v4.4s, #16 -; CHECK-NEXT: add v20.4s, v6.4s, v1.4s -; CHECK-NEXT: add v1.4s, v4.4s, v1.4s -; CHECK-NEXT: and v5.16b, v5.16b, v3.16b -; CHECK-NEXT: and v7.16b, v7.16b, v3.16b -; CHECK-NEXT: and v16.16b, v16.16b, v3.16b -; CHECK-NEXT: and v3.16b, v18.16b, v3.16b -; CHECK-NEXT: fcmeq v18.4s, v0.4s, v0.4s -; CHECK-NEXT: orr v0.4s, #64, lsl #16 -; CHECK-NEXT: add v5.4s, v5.4s, v17.4s +; CHECK-NEXT: ushr v17.4s, v4.4s, #16 +; CHECK-NEXT: add v19.4s, v0.4s, v3.4s +; CHECK-NEXT: add v18.4s, v2.4s, v3.4s +; CHECK-NEXT: add v20.4s, v6.4s, v3.4s +; CHECK-NEXT: add v3.4s, v4.4s, v3.4s +; CHECK-NEXT: and v7.16b, v7.16b, v1.16b +; CHECK-NEXT: and v5.16b, v5.16b, v1.16b +; CHECK-NEXT: and v16.16b, v16.16b, v1.16b +; CHECK-NEXT: and v1.16b, v17.16b, v1.16b ; CHECK-NEXT: fcmeq v17.4s, v2.4s, v2.4s +; CHECK-NEXT: orr v2.4s, #64, lsl #16 ; CHECK-NEXT: add v7.4s, v7.4s, v19.4s ; CHECK-NEXT: fcmeq v19.4s, v6.4s, v6.4s -; CHECK-NEXT: orr v2.4s, #64, lsl #16 -; CHECK-NEXT: add v16.4s, v16.4s, v20.4s -; CHECK-NEXT: add v1.4s, v3.4s, v1.4s +; CHECK-NEXT: add v5.4s, v5.4s, v18.4s +; CHECK-NEXT: fcmeq v18.4s, v0.4s, v0.4s +; CHECK-NEXT: add v1.4s, v1.4s, v3.4s ; CHECK-NEXT: fcmeq v3.4s, v4.4s, v4.4s +; CHECK-NEXT: add v16.4s, v16.4s, v20.4s +; CHECK-NEXT: orr v0.4s, #64, lsl #16 ; CHECK-NEXT: orr v6.4s, #64, lsl #16 ; CHECK-NEXT: orr v4.4s, #64, lsl #16 ; CHECK-NEXT: bit v2.16b, v5.16b, v17.16b -; CHECK-NEXT: bit v0.16b, v7.16b, v18.16b ; CHECK-NEXT: mov v5.16b, v19.16b +; CHECK-NEXT: bit v0.16b, v7.16b, v18.16b ; CHECK-NEXT: bif v1.16b, v4.16b, v3.16b ; CHECK-NEXT: bsl v5.16b, v16.16b, v6.16b ; CHECK-NEXT: uzp2 v0.8h, v0.8h, v2.8h @@ -6776,51 +6776,51 @@ entry: define <16 x bfloat> @utofp_v16i64_v16bf16(<16 x i64> %a) { ; CHECK-LABEL: utofp_v16i64_v16bf16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ucvtf v2.2d, v2.2d ; CHECK-NEXT: ucvtf v0.2d, v0.2d +; CHECK-NEXT: ucvtf v2.2d, v2.2d ; CHECK-NEXT: ucvtf v6.2d, v6.2d ; CHECK-NEXT: ucvtf v4.2d, v4.2d -; CHECK-NEXT: ucvtf v3.2d, v3.2d ; CHECK-NEXT: ucvtf v1.2d, v1.2d +; CHECK-NEXT: ucvtf v3.2d, v3.2d ; CHECK-NEXT: ucvtf v7.2d, v7.2d ; CHECK-NEXT: ucvtf v5.2d, v5.2d -; CHECK-NEXT: fcvtn v2.2s, v2.2d ; CHECK-NEXT: fcvtn v0.2s, v0.2d +; CHECK-NEXT: fcvtn v2.2s, v2.2d ; CHECK-NEXT: fcvtn v6.2s, v6.2d ; CHECK-NEXT: fcvtn v4.2s, v4.2d -; CHECK-NEXT: fcvtn2 v2.4s, v3.2d ; CHECK-NEXT: fcvtn2 v0.4s, v1.2d -; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: fcvtn2 v2.4s, v3.2d ; CHECK-NEXT: fcvtn2 v6.4s, v7.2d ; CHECK-NEXT: fcvtn2 v4.4s, v5.2d -; CHECK-NEXT: movi v3.4s, #1 -; CHECK-NEXT: ushr v5.4s, v2.4s, #16 +; CHECK-NEXT: movi v1.4s, #1 +; CHECK-NEXT: movi v3.4s, #127, msl #8 ; CHECK-NEXT: ushr v7.4s, v0.4s, #16 -; CHECK-NEXT: add v17.4s, v2.4s, v1.4s -; CHECK-NEXT: add v19.4s, v0.4s, v1.4s +; CHECK-NEXT: ushr v5.4s, v2.4s, #16 ; CHECK-NEXT: ushr v16.4s, v6.4s, #16 -; CHECK-NEXT: ushr v18.4s, v4.4s, #16 -; CHECK-NEXT: add v20.4s, v6.4s, v1.4s -; CHECK-NEXT: add v1.4s, v4.4s, v1.4s -; CHECK-NEXT: and v5.16b, v5.16b, v3.16b -; CHECK-NEXT: and v7.16b, v7.16b, v3.16b -; CHECK-NEXT: and v16.16b, v16.16b, v3.16b -; CHECK-NEXT: and v3.16b, v18.16b, v3.16b -; CHECK-NEXT: fcmeq v18.4s, v0.4s, v0.4s -; CHECK-NEXT: orr v0.4s, #64, lsl #16 -; CHECK-NEXT: add v5.4s, v5.4s, v17.4s +; CHECK-NEXT: ushr v17.4s, v4.4s, #16 +; CHECK-NEXT: add v19.4s, v0.4s, v3.4s +; CHECK-NEXT: add v18.4s, v2.4s, v3.4s +; CHECK-NEXT: add v20.4s, v6.4s, v3.4s +; CHECK-NEXT: add v3.4s, v4.4s, v3.4s +; CHECK-NEXT: and v7.16b, v7.16b, v1.16b +; CHECK-NEXT: and v5.16b, v5.16b, v1.16b +; CHECK-NEXT: and v16.16b, v16.16b, v1.16b +; CHECK-NEXT: and v1.16b, v17.16b, v1.16b ; CHECK-NEXT: fcmeq v17.4s, v2.4s, v2.4s +; CHECK-NEXT: orr v2.4s, #64, lsl #16 ; CHECK-NEXT: add v7.4s, v7.4s, v19.4s ; CHECK-NEXT: fcmeq v19.4s, v6.4s, v6.4s -; CHECK-NEXT: orr v2.4s, #64, lsl #16 -; CHECK-NEXT: add v16.4s, v16.4s, v20.4s -; CHECK-NEXT: add v1.4s, v3.4s, v1.4s +; CHECK-NEXT: add v5.4s, v5.4s, v18.4s +; CHECK-NEXT: fcmeq v18.4s, v0.4s, v0.4s +; CHECK-NEXT: add v1.4s, v1.4s, v3.4s ; CHECK-NEXT: fcmeq v3.4s, v4.4s, v4.4s +; CHECK-NEXT: add v16.4s, v16.4s, v20.4s +; CHECK-NEXT: orr v0.4s, #64, lsl #16 ; CHECK-NEXT: orr v6.4s, #64, lsl #16 ; CHECK-NEXT: orr v4.4s, #64, lsl #16 ; CHECK-NEXT: bit v2.16b, v5.16b, v17.16b -; CHECK-NEXT: bit v0.16b, v7.16b, v18.16b ; CHECK-NEXT: mov v5.16b, v19.16b +; CHECK-NEXT: bit v0.16b, v7.16b, v18.16b ; CHECK-NEXT: bif v1.16b, v4.16b, v3.16b ; CHECK-NEXT: bsl v5.16b, v16.16b, v6.16b ; CHECK-NEXT: uzp2 v0.8h, v0.8h, v2.8h @@ -6834,107 +6834,107 @@ entry: define <32 x bfloat> @stofp_v32i64_v32bf16(<32 x i64> %a) { ; CHECK-LABEL: stofp_v32i64_v32bf16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: scvtf v16.2d, v2.2d -; CHECK-NEXT: scvtf v17.2d, v0.2d -; CHECK-NEXT: scvtf v18.2d, v3.2d -; CHECK-NEXT: scvtf v19.2d, v6.2d -; CHECK-NEXT: ldp q24, q23, [sp, #96] -; CHECK-NEXT: scvtf v21.2d, v1.2d -; CHECK-NEXT: scvtf v22.2d, v4.2d +; CHECK-NEXT: scvtf v17.2d, v2.2d +; CHECK-NEXT: scvtf v18.2d, v0.2d +; CHECK-NEXT: scvtf v19.2d, v3.2d +; CHECK-NEXT: scvtf v3.2d, v6.2d +; CHECK-NEXT: ldp q21, q20, [sp, #32] +; CHECK-NEXT: scvtf v4.2d, v4.2d ; CHECK-NEXT: scvtf v6.2d, v7.2d -; CHECK-NEXT: scvtf v7.2d, v5.2d -; CHECK-NEXT: movi v3.4s, #127, msl #8 -; CHECK-NEXT: movi v2.4s, #1 -; CHECK-NEXT: fcvtn v0.2s, v16.2d -; CHECK-NEXT: ldp q20, q16, [sp, #32] -; CHECK-NEXT: fcvtn v1.2s, v17.2d -; CHECK-NEXT: ldp q5, q17, [sp] -; CHECK-NEXT: fcvtn v4.2s, v19.2d -; CHECK-NEXT: scvtf v23.2d, v23.2d +; CHECK-NEXT: scvtf v5.2d, v5.2d +; CHECK-NEXT: ldp q24, q23, [sp, #64] +; CHECK-NEXT: movi v16.4s, #1 +; CHECK-NEXT: fcvtn v0.2s, v17.2d +; CHECK-NEXT: scvtf v17.2d, v1.2d +; CHECK-NEXT: fcvtn v1.2s, v18.2d +; CHECK-NEXT: fcvtn v3.2s, v3.2d +; CHECK-NEXT: ldp q18, q7, [sp] +; CHECK-NEXT: scvtf v21.2d, v21.2d +; CHECK-NEXT: fcvtn v4.2s, v4.2d +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: scvtf v20.2d, v20.2d -; CHECK-NEXT: scvtf v16.2d, v16.2d -; CHECK-NEXT: fcvtn2 v0.4s, v18.2d -; CHECK-NEXT: ldp q19, q18, [sp, #64] -; CHECK-NEXT: scvtf v25.2d, v5.2d -; CHECK-NEXT: fcvtn v5.2s, v22.2d -; CHECK-NEXT: fcvtn2 v1.4s, v21.2d -; CHECK-NEXT: scvtf v21.2d, v24.2d -; CHECK-NEXT: scvtf v17.2d, v17.2d -; CHECK-NEXT: fcvtn2 v4.4s, v6.2d +; CHECK-NEXT: fcvtn2 v0.4s, v19.2d +; CHECK-NEXT: ldp q22, q19, [sp, #96] +; CHECK-NEXT: fcvtn2 v1.4s, v17.2d +; CHECK-NEXT: fcvtn2 v3.4s, v6.2d +; CHECK-NEXT: scvtf v18.2d, v18.2d +; CHECK-NEXT: scvtf v17.2d, v24.2d +; CHECK-NEXT: fcvtn v6.2s, v21.2d +; CHECK-NEXT: fcvtn2 v4.4s, v5.2d +; CHECK-NEXT: scvtf v22.2d, v22.2d +; CHECK-NEXT: scvtf v21.2d, v23.2d +; CHECK-NEXT: scvtf v7.2d, v7.2d +; CHECK-NEXT: ushr v24.4s, v0.4s, #16 +; CHECK-NEXT: add v5.4s, v0.4s, v2.4s ; CHECK-NEXT: scvtf v19.2d, v19.2d -; CHECK-NEXT: scvtf v6.2d, v18.2d -; CHECK-NEXT: fcvtn v18.2s, v20.2d -; CHECK-NEXT: ushr v22.4s, v0.4s, #16 -; CHECK-NEXT: add v20.4s, v0.4s, v3.4s -; CHECK-NEXT: fcvtn2 v5.4s, v7.2d -; CHECK-NEXT: fcvtn v24.2s, v25.2d -; CHECK-NEXT: ushr v7.4s, v1.4s, #16 -; CHECK-NEXT: fcvtn v21.2s, v21.2d -; CHECK-NEXT: add v26.4s, v1.4s, v3.4s -; CHECK-NEXT: ushr v27.4s, v4.4s, #16 -; CHECK-NEXT: fcvtn v19.2s, v19.2d -; CHECK-NEXT: fcvtn2 v18.4s, v16.2d -; CHECK-NEXT: and v22.16b, v22.16b, v2.16b -; CHECK-NEXT: and v7.16b, v7.16b, v2.16b -; CHECK-NEXT: fcmeq v25.4s, v0.4s, v0.4s +; CHECK-NEXT: ushr v23.4s, v1.4s, #16 +; CHECK-NEXT: ushr v25.4s, v3.4s, #16 +; CHECK-NEXT: fcvtn v18.2s, v18.2d +; CHECK-NEXT: fcvtn2 v6.4s, v20.2d +; CHECK-NEXT: add v26.4s, v1.4s, v2.4s +; CHECK-NEXT: fcvtn v17.2s, v17.2d +; CHECK-NEXT: and v24.16b, v24.16b, v16.16b +; CHECK-NEXT: fcvtn v22.2s, v22.2d +; CHECK-NEXT: fcmeq v20.4s, v0.4s, v0.4s +; CHECK-NEXT: and v23.16b, v23.16b, v16.16b ; CHECK-NEXT: orr v0.4s, #64, lsl #16 -; CHECK-NEXT: ushr v28.4s, v5.4s, #16 -; CHECK-NEXT: fcvtn2 v24.4s, v17.2d -; CHECK-NEXT: add v17.4s, v5.4s, v3.4s -; CHECK-NEXT: fcvtn2 v21.4s, v23.2d -; CHECK-NEXT: and v16.16b, v27.16b, v2.16b -; CHECK-NEXT: add v20.4s, v22.4s, v20.4s -; CHECK-NEXT: fcvtn2 v19.4s, v6.2d -; CHECK-NEXT: add v7.4s, v7.4s, v26.4s -; CHECK-NEXT: ushr v26.4s, v18.4s, #16 -; CHECK-NEXT: and v23.16b, v28.16b, v2.16b -; CHECK-NEXT: add v22.4s, v4.4s, v3.4s -; CHECK-NEXT: fcmeq v6.4s, v1.4s, v1.4s -; CHECK-NEXT: ushr v27.4s, v24.4s, #16 -; CHECK-NEXT: add v30.4s, v24.4s, v3.4s +; CHECK-NEXT: fcmeq v27.4s, v3.4s, v3.4s +; CHECK-NEXT: fcvtn2 v18.4s, v7.2d +; CHECK-NEXT: add v7.4s, v3.4s, v2.4s +; CHECK-NEXT: orr v3.4s, #64, lsl #16 +; CHECK-NEXT: add v5.4s, v24.4s, v5.4s +; CHECK-NEXT: and v24.16b, v25.16b, v16.16b +; CHECK-NEXT: ushr v25.4s, v4.4s, #16 +; CHECK-NEXT: fcvtn2 v22.4s, v19.2d +; CHECK-NEXT: add v19.4s, v23.4s, v26.4s +; CHECK-NEXT: ushr v26.4s, v6.4s, #16 +; CHECK-NEXT: fcvtn2 v17.4s, v21.2d +; CHECK-NEXT: fcmeq v21.4s, v1.4s, v1.4s ; CHECK-NEXT: orr v1.4s, #64, lsl #16 -; CHECK-NEXT: ushr v28.4s, v21.4s, #16 -; CHECK-NEXT: add v31.4s, v21.4s, v3.4s -; CHECK-NEXT: and v26.16b, v26.16b, v2.16b -; CHECK-NEXT: add v17.4s, v23.4s, v17.4s -; CHECK-NEXT: add v23.4s, v18.4s, v3.4s -; CHECK-NEXT: ushr v29.4s, v19.4s, #16 -; CHECK-NEXT: and v27.16b, v27.16b, v2.16b -; CHECK-NEXT: add v3.4s, v19.4s, v3.4s -; CHECK-NEXT: add v16.4s, v16.4s, v22.4s -; CHECK-NEXT: and v28.16b, v28.16b, v2.16b -; CHECK-NEXT: fcmeq v22.4s, v4.4s, v4.4s -; CHECK-NEXT: orr v4.4s, #64, lsl #16 -; CHECK-NEXT: and v2.16b, v29.16b, v2.16b -; CHECK-NEXT: fcmeq v29.4s, v5.4s, v5.4s -; CHECK-NEXT: orr v5.4s, #64, lsl #16 -; CHECK-NEXT: add v23.4s, v26.4s, v23.4s -; CHECK-NEXT: fcmeq v26.4s, v18.4s, v18.4s -; CHECK-NEXT: add v27.4s, v27.4s, v30.4s -; CHECK-NEXT: fcmeq v30.4s, v24.4s, v24.4s +; CHECK-NEXT: and v23.16b, v25.16b, v16.16b +; CHECK-NEXT: add v25.4s, v4.4s, v2.4s +; CHECK-NEXT: add v7.4s, v24.4s, v7.4s +; CHECK-NEXT: ushr v24.4s, v18.4s, #16 +; CHECK-NEXT: add v30.4s, v18.4s, v2.4s +; CHECK-NEXT: bit v0.16b, v5.16b, v20.16b +; CHECK-NEXT: ushr v28.4s, v22.4s, #16 +; CHECK-NEXT: add v31.4s, v22.4s, v2.4s +; CHECK-NEXT: add v23.4s, v23.4s, v25.4s +; CHECK-NEXT: and v25.16b, v26.16b, v16.16b +; CHECK-NEXT: add v26.4s, v6.4s, v2.4s +; CHECK-NEXT: ushr v29.4s, v17.4s, #16 +; CHECK-NEXT: and v24.16b, v24.16b, v16.16b +; CHECK-NEXT: add v2.4s, v17.4s, v2.4s +; CHECK-NEXT: and v28.16b, v28.16b, v16.16b +; CHECK-NEXT: bit v3.16b, v7.16b, v27.16b +; CHECK-NEXT: bit v1.16b, v19.16b, v21.16b +; CHECK-NEXT: add v25.4s, v25.4s, v26.4s +; CHECK-NEXT: fcmeq v26.4s, v6.4s, v6.4s +; CHECK-NEXT: orr v6.4s, #64, lsl #16 +; CHECK-NEXT: and v16.16b, v29.16b, v16.16b +; CHECK-NEXT: add v24.4s, v24.4s, v30.4s +; CHECK-NEXT: fcmeq v30.4s, v18.4s, v18.4s ; CHECK-NEXT: add v28.4s, v28.4s, v31.4s -; CHECK-NEXT: fcmeq v31.4s, v21.4s, v21.4s -; CHECK-NEXT: add v2.4s, v2.4s, v3.4s -; CHECK-NEXT: fcmeq v3.4s, v19.4s, v19.4s +; CHECK-NEXT: fcmeq v31.4s, v22.4s, v22.4s +; CHECK-NEXT: fcmeq v29.4s, v4.4s, v4.4s +; CHECK-NEXT: orr v4.4s, #64, lsl #16 ; CHECK-NEXT: orr v18.4s, #64, lsl #16 -; CHECK-NEXT: orr v24.4s, #64, lsl #16 -; CHECK-NEXT: orr v21.4s, #64, lsl #16 -; CHECK-NEXT: orr v19.4s, #64, lsl #16 -; CHECK-NEXT: bit v1.16b, v7.16b, v6.16b -; CHECK-NEXT: bit v4.16b, v16.16b, v22.16b -; CHECK-NEXT: mov v6.16b, v26.16b -; CHECK-NEXT: mov v7.16b, v30.16b -; CHECK-NEXT: mov v16.16b, v31.16b -; CHECK-NEXT: bit v0.16b, v20.16b, v25.16b -; CHECK-NEXT: bit v5.16b, v17.16b, v29.16b -; CHECK-NEXT: bsl v3.16b, v2.16b, v19.16b -; CHECK-NEXT: bsl v6.16b, v23.16b, v18.16b -; CHECK-NEXT: bsl v7.16b, v27.16b, v24.16b -; CHECK-NEXT: bsl v16.16b, v28.16b, v21.16b +; CHECK-NEXT: orr v22.4s, #64, lsl #16 +; CHECK-NEXT: mov v5.16b, v26.16b +; CHECK-NEXT: add v2.4s, v16.4s, v2.4s +; CHECK-NEXT: fcmeq v16.4s, v17.4s, v17.4s +; CHECK-NEXT: orr v17.4s, #64, lsl #16 ; CHECK-NEXT: uzp2 v0.8h, v1.8h, v0.8h -; CHECK-NEXT: uzp2 v1.8h, v5.8h, v4.8h -; CHECK-NEXT: uzp2 v2.8h, v7.8h, v6.8h -; CHECK-NEXT: uzp2 v3.8h, v3.8h, v16.8h +; CHECK-NEXT: mov v7.16b, v31.16b +; CHECK-NEXT: bit v4.16b, v23.16b, v29.16b +; CHECK-NEXT: bsl v5.16b, v25.16b, v6.16b +; CHECK-NEXT: mov v6.16b, v30.16b +; CHECK-NEXT: bsl v16.16b, v2.16b, v17.16b +; CHECK-NEXT: bsl v7.16b, v28.16b, v22.16b +; CHECK-NEXT: bsl v6.16b, v24.16b, v18.16b +; CHECK-NEXT: uzp2 v1.8h, v4.8h, v3.8h +; CHECK-NEXT: uzp2 v3.8h, v16.8h, v7.8h +; CHECK-NEXT: uzp2 v2.8h, v6.8h, v5.8h ; CHECK-NEXT: ret entry: %c = sitofp <32 x i64> %a to <32 x bfloat> @@ -6944,107 +6944,107 @@ entry: define <32 x bfloat> @utofp_v32i64_v32bf16(<32 x i64> %a) { ; CHECK-LABEL: utofp_v32i64_v32bf16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ucvtf v16.2d, v2.2d -; CHECK-NEXT: ucvtf v17.2d, v0.2d -; CHECK-NEXT: ucvtf v18.2d, v3.2d -; CHECK-NEXT: ucvtf v19.2d, v6.2d -; CHECK-NEXT: ldp q24, q23, [sp, #96] -; CHECK-NEXT: ucvtf v21.2d, v1.2d -; CHECK-NEXT: ucvtf v22.2d, v4.2d +; CHECK-NEXT: ucvtf v17.2d, v2.2d +; CHECK-NEXT: ucvtf v18.2d, v0.2d +; CHECK-NEXT: ucvtf v19.2d, v3.2d +; CHECK-NEXT: ucvtf v3.2d, v6.2d +; CHECK-NEXT: ldp q21, q20, [sp, #32] +; CHECK-NEXT: ucvtf v4.2d, v4.2d ; CHECK-NEXT: ucvtf v6.2d, v7.2d -; CHECK-NEXT: ucvtf v7.2d, v5.2d -; CHECK-NEXT: movi v3.4s, #127, msl #8 -; CHECK-NEXT: movi v2.4s, #1 -; CHECK-NEXT: fcvtn v0.2s, v16.2d -; CHECK-NEXT: ldp q20, q16, [sp, #32] -; CHECK-NEXT: fcvtn v1.2s, v17.2d -; CHECK-NEXT: ldp q5, q17, [sp] -; CHECK-NEXT: fcvtn v4.2s, v19.2d -; CHECK-NEXT: ucvtf v23.2d, v23.2d +; CHECK-NEXT: ucvtf v5.2d, v5.2d +; CHECK-NEXT: ldp q24, q23, [sp, #64] +; CHECK-NEXT: movi v16.4s, #1 +; CHECK-NEXT: fcvtn v0.2s, v17.2d +; CHECK-NEXT: ucvtf v17.2d, v1.2d +; CHECK-NEXT: fcvtn v1.2s, v18.2d +; CHECK-NEXT: fcvtn v3.2s, v3.2d +; CHECK-NEXT: ldp q18, q7, [sp] +; CHECK-NEXT: ucvtf v21.2d, v21.2d +; CHECK-NEXT: fcvtn v4.2s, v4.2d +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: ucvtf v20.2d, v20.2d -; CHECK-NEXT: ucvtf v16.2d, v16.2d -; CHECK-NEXT: fcvtn2 v0.4s, v18.2d -; CHECK-NEXT: ldp q19, q18, [sp, #64] -; CHECK-NEXT: ucvtf v25.2d, v5.2d -; CHECK-NEXT: fcvtn v5.2s, v22.2d -; CHECK-NEXT: fcvtn2 v1.4s, v21.2d -; CHECK-NEXT: ucvtf v21.2d, v24.2d -; CHECK-NEXT: ucvtf v17.2d, v17.2d -; CHECK-NEXT: fcvtn2 v4.4s, v6.2d +; CHECK-NEXT: fcvtn2 v0.4s, v19.2d +; CHECK-NEXT: ldp q22, q19, [sp, #96] +; CHECK-NEXT: fcvtn2 v1.4s, v17.2d +; CHECK-NEXT: fcvtn2 v3.4s, v6.2d +; CHECK-NEXT: ucvtf v18.2d, v18.2d +; CHECK-NEXT: ucvtf v17.2d, v24.2d +; CHECK-NEXT: fcvtn v6.2s, v21.2d +; CHECK-NEXT: fcvtn2 v4.4s, v5.2d +; CHECK-NEXT: ucvtf v22.2d, v22.2d +; CHECK-NEXT: ucvtf v21.2d, v23.2d +; CHECK-NEXT: ucvtf v7.2d, v7.2d +; CHECK-NEXT: ushr v24.4s, v0.4s, #16 +; CHECK-NEXT: add v5.4s, v0.4s, v2.4s ; CHECK-NEXT: ucvtf v19.2d, v19.2d -; CHECK-NEXT: ucvtf v6.2d, v18.2d -; CHECK-NEXT: fcvtn v18.2s, v20.2d -; CHECK-NEXT: ushr v22.4s, v0.4s, #16 -; CHECK-NEXT: add v20.4s, v0.4s, v3.4s -; CHECK-NEXT: fcvtn2 v5.4s, v7.2d -; CHECK-NEXT: fcvtn v24.2s, v25.2d -; CHECK-NEXT: ushr v7.4s, v1.4s, #16 -; CHECK-NEXT: fcvtn v21.2s, v21.2d -; CHECK-NEXT: add v26.4s, v1.4s, v3.4s -; CHECK-NEXT: ushr v27.4s, v4.4s, #16 -; CHECK-NEXT: fcvtn v19.2s, v19.2d -; CHECK-NEXT: fcvtn2 v18.4s, v16.2d -; CHECK-NEXT: and v22.16b, v22.16b, v2.16b -; CHECK-NEXT: and v7.16b, v7.16b, v2.16b -; CHECK-NEXT: fcmeq v25.4s, v0.4s, v0.4s +; CHECK-NEXT: ushr v23.4s, v1.4s, #16 +; CHECK-NEXT: ushr v25.4s, v3.4s, #16 +; CHECK-NEXT: fcvtn v18.2s, v18.2d +; CHECK-NEXT: fcvtn2 v6.4s, v20.2d +; CHECK-NEXT: add v26.4s, v1.4s, v2.4s +; CHECK-NEXT: fcvtn v17.2s, v17.2d +; CHECK-NEXT: and v24.16b, v24.16b, v16.16b +; CHECK-NEXT: fcvtn v22.2s, v22.2d +; CHECK-NEXT: fcmeq v20.4s, v0.4s, v0.4s +; CHECK-NEXT: and v23.16b, v23.16b, v16.16b ; CHECK-NEXT: orr v0.4s, #64, lsl #16 -; CHECK-NEXT: ushr v28.4s, v5.4s, #16 -; CHECK-NEXT: fcvtn2 v24.4s, v17.2d -; CHECK-NEXT: add v17.4s, v5.4s, v3.4s -; CHECK-NEXT: fcvtn2 v21.4s, v23.2d -; CHECK-NEXT: and v16.16b, v27.16b, v2.16b -; CHECK-NEXT: add v20.4s, v22.4s, v20.4s -; CHECK-NEXT: fcvtn2 v19.4s, v6.2d -; CHECK-NEXT: add v7.4s, v7.4s, v26.4s -; CHECK-NEXT: ushr v26.4s, v18.4s, #16 -; CHECK-NEXT: and v23.16b, v28.16b, v2.16b -; CHECK-NEXT: add v22.4s, v4.4s, v3.4s -; CHECK-NEXT: fcmeq v6.4s, v1.4s, v1.4s -; CHECK-NEXT: ushr v27.4s, v24.4s, #16 -; CHECK-NEXT: add v30.4s, v24.4s, v3.4s +; CHECK-NEXT: fcmeq v27.4s, v3.4s, v3.4s +; CHECK-NEXT: fcvtn2 v18.4s, v7.2d +; CHECK-NEXT: add v7.4s, v3.4s, v2.4s +; CHECK-NEXT: orr v3.4s, #64, lsl #16 +; CHECK-NEXT: add v5.4s, v24.4s, v5.4s +; CHECK-NEXT: and v24.16b, v25.16b, v16.16b +; CHECK-NEXT: ushr v25.4s, v4.4s, #16 +; CHECK-NEXT: fcvtn2 v22.4s, v19.2d +; CHECK-NEXT: add v19.4s, v23.4s, v26.4s +; CHECK-NEXT: ushr v26.4s, v6.4s, #16 +; CHECK-NEXT: fcvtn2 v17.4s, v21.2d +; CHECK-NEXT: fcmeq v21.4s, v1.4s, v1.4s ; CHECK-NEXT: orr v1.4s, #64, lsl #16 -; CHECK-NEXT: ushr v28.4s, v21.4s, #16 -; CHECK-NEXT: add v31.4s, v21.4s, v3.4s -; CHECK-NEXT: and v26.16b, v26.16b, v2.16b -; CHECK-NEXT: add v17.4s, v23.4s, v17.4s -; CHECK-NEXT: add v23.4s, v18.4s, v3.4s -; CHECK-NEXT: ushr v29.4s, v19.4s, #16 -; CHECK-NEXT: and v27.16b, v27.16b, v2.16b -; CHECK-NEXT: add v3.4s, v19.4s, v3.4s -; CHECK-NEXT: add v16.4s, v16.4s, v22.4s -; CHECK-NEXT: and v28.16b, v28.16b, v2.16b -; CHECK-NEXT: fcmeq v22.4s, v4.4s, v4.4s -; CHECK-NEXT: orr v4.4s, #64, lsl #16 -; CHECK-NEXT: and v2.16b, v29.16b, v2.16b -; CHECK-NEXT: fcmeq v29.4s, v5.4s, v5.4s -; CHECK-NEXT: orr v5.4s, #64, lsl #16 -; CHECK-NEXT: add v23.4s, v26.4s, v23.4s -; CHECK-NEXT: fcmeq v26.4s, v18.4s, v18.4s -; CHECK-NEXT: add v27.4s, v27.4s, v30.4s -; CHECK-NEXT: fcmeq v30.4s, v24.4s, v24.4s +; CHECK-NEXT: and v23.16b, v25.16b, v16.16b +; CHECK-NEXT: add v25.4s, v4.4s, v2.4s +; CHECK-NEXT: add v7.4s, v24.4s, v7.4s +; CHECK-NEXT: ushr v24.4s, v18.4s, #16 +; CHECK-NEXT: add v30.4s, v18.4s, v2.4s +; CHECK-NEXT: bit v0.16b, v5.16b, v20.16b +; CHECK-NEXT: ushr v28.4s, v22.4s, #16 +; CHECK-NEXT: add v31.4s, v22.4s, v2.4s +; CHECK-NEXT: add v23.4s, v23.4s, v25.4s +; CHECK-NEXT: and v25.16b, v26.16b, v16.16b +; CHECK-NEXT: add v26.4s, v6.4s, v2.4s +; CHECK-NEXT: ushr v29.4s, v17.4s, #16 +; CHECK-NEXT: and v24.16b, v24.16b, v16.16b +; CHECK-NEXT: add v2.4s, v17.4s, v2.4s +; CHECK-NEXT: and v28.16b, v28.16b, v16.16b +; CHECK-NEXT: bit v3.16b, v7.16b, v27.16b +; CHECK-NEXT: bit v1.16b, v19.16b, v21.16b +; CHECK-NEXT: add v25.4s, v25.4s, v26.4s +; CHECK-NEXT: fcmeq v26.4s, v6.4s, v6.4s +; CHECK-NEXT: orr v6.4s, #64, lsl #16 +; CHECK-NEXT: and v16.16b, v29.16b, v16.16b +; CHECK-NEXT: add v24.4s, v24.4s, v30.4s +; CHECK-NEXT: fcmeq v30.4s, v18.4s, v18.4s ; CHECK-NEXT: add v28.4s, v28.4s, v31.4s -; CHECK-NEXT: fcmeq v31.4s, v21.4s, v21.4s -; CHECK-NEXT: add v2.4s, v2.4s, v3.4s -; CHECK-NEXT: fcmeq v3.4s, v19.4s, v19.4s +; CHECK-NEXT: fcmeq v31.4s, v22.4s, v22.4s +; CHECK-NEXT: fcmeq v29.4s, v4.4s, v4.4s +; CHECK-NEXT: orr v4.4s, #64, lsl #16 ; CHECK-NEXT: orr v18.4s, #64, lsl #16 -; CHECK-NEXT: orr v24.4s, #64, lsl #16 -; CHECK-NEXT: orr v21.4s, #64, lsl #16 -; CHECK-NEXT: orr v19.4s, #64, lsl #16 -; CHECK-NEXT: bit v1.16b, v7.16b, v6.16b -; CHECK-NEXT: bit v4.16b, v16.16b, v22.16b -; CHECK-NEXT: mov v6.16b, v26.16b -; CHECK-NEXT: mov v7.16b, v30.16b -; CHECK-NEXT: mov v16.16b, v31.16b -; CHECK-NEXT: bit v0.16b, v20.16b, v25.16b -; CHECK-NEXT: bit v5.16b, v17.16b, v29.16b -; CHECK-NEXT: bsl v3.16b, v2.16b, v19.16b -; CHECK-NEXT: bsl v6.16b, v23.16b, v18.16b -; CHECK-NEXT: bsl v7.16b, v27.16b, v24.16b -; CHECK-NEXT: bsl v16.16b, v28.16b, v21.16b +; CHECK-NEXT: orr v22.4s, #64, lsl #16 +; CHECK-NEXT: mov v5.16b, v26.16b +; CHECK-NEXT: add v2.4s, v16.4s, v2.4s +; CHECK-NEXT: fcmeq v16.4s, v17.4s, v17.4s +; CHECK-NEXT: orr v17.4s, #64, lsl #16 ; CHECK-NEXT: uzp2 v0.8h, v1.8h, v0.8h -; CHECK-NEXT: uzp2 v1.8h, v5.8h, v4.8h -; CHECK-NEXT: uzp2 v2.8h, v7.8h, v6.8h -; CHECK-NEXT: uzp2 v3.8h, v3.8h, v16.8h +; CHECK-NEXT: mov v7.16b, v31.16b +; CHECK-NEXT: bit v4.16b, v23.16b, v29.16b +; CHECK-NEXT: bsl v5.16b, v25.16b, v6.16b +; CHECK-NEXT: mov v6.16b, v30.16b +; CHECK-NEXT: bsl v16.16b, v2.16b, v17.16b +; CHECK-NEXT: bsl v7.16b, v28.16b, v22.16b +; CHECK-NEXT: bsl v6.16b, v24.16b, v18.16b +; CHECK-NEXT: uzp2 v1.8h, v4.8h, v3.8h +; CHECK-NEXT: uzp2 v3.8h, v16.8h, v7.8h +; CHECK-NEXT: uzp2 v2.8h, v6.8h, v5.8h ; CHECK-NEXT: ret entry: %c = uitofp <32 x i64> %a to <32 x bfloat> @@ -7059,9 +7059,9 @@ define <2 x bfloat> @stofp_v2i32_v2bf16(<2 x i32> %a) { ; CHECK-NEXT: scvtf v0.4s, v0.4s ; CHECK-NEXT: ushr v2.4s, v0.4s, #16 ; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v2.4s ; CHECK-NEXT: ret entry: %c = sitofp <2 x i32> %a to <2 x bfloat> @@ -7076,9 +7076,9 @@ define <2 x bfloat> @utofp_v2i32_v2bf16(<2 x i32> %a) { ; CHECK-NEXT: ucvtf v0.4s, v0.4s ; CHECK-NEXT: ushr v2.4s, v0.4s, #16 ; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v2.4s ; CHECK-NEXT: ret entry: %c = uitofp <2 x i32> %a to <2 x bfloat> @@ -7092,9 +7092,9 @@ define <3 x bfloat> @stofp_v3i32_v3bf16(<3 x i32> %a) { ; CHECK-NEXT: movi v1.4s, #1 ; CHECK-NEXT: ushr v2.4s, v0.4s, #16 ; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v2.4s ; CHECK-NEXT: ret entry: %c = sitofp <3 x i32> %a to <3 x bfloat> @@ -7108,9 +7108,9 @@ define <3 x bfloat> @utofp_v3i32_v3bf16(<3 x i32> %a) { ; CHECK-NEXT: movi v1.4s, #1 ; CHECK-NEXT: ushr v2.4s, v0.4s, #16 ; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v2.4s ; CHECK-NEXT: ret entry: %c = uitofp <3 x i32> %a to <3 x bfloat> @@ -7124,9 +7124,9 @@ define <4 x bfloat> @stofp_v4i32_v4bf16(<4 x i32> %a) { ; CHECK-NEXT: movi v1.4s, #1 ; CHECK-NEXT: ushr v2.4s, v0.4s, #16 ; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v2.4s ; CHECK-NEXT: ret entry: %c = sitofp <4 x i32> %a to <4 x bfloat> @@ -7140,9 +7140,9 @@ define <4 x bfloat> @utofp_v4i32_v4bf16(<4 x i32> %a) { ; CHECK-NEXT: movi v1.4s, #1 ; CHECK-NEXT: ushr v2.4s, v0.4s, #16 ; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v2.4s ; CHECK-NEXT: ret entry: %c = uitofp <4 x i32> %a to <4 x bfloat> @@ -7155,15 +7155,15 @@ define <8 x bfloat> @stofp_v8i32_v8bf16(<8 x i32> %a) { ; CHECK-NEXT: scvtf v0.4s, v0.4s ; CHECK-NEXT: movi v2.4s, #1 ; CHECK-NEXT: scvtf v1.4s, v1.4s +; CHECK-NEXT: movi v5.4s, #127, msl #8 ; CHECK-NEXT: ushr v3.4s, v0.4s, #16 ; CHECK-NEXT: ushr v4.4s, v1.4s, #16 ; CHECK-NEXT: and v3.16b, v3.16b, v2.16b ; CHECK-NEXT: and v2.16b, v4.16b, v2.16b ; CHECK-NEXT: add v0.4s, v3.4s, v0.4s -; CHECK-NEXT: movi v3.4s, #127, msl #8 ; CHECK-NEXT: add v1.4s, v2.4s, v1.4s -; CHECK-NEXT: addhn v0.4h, v0.4s, v3.4s -; CHECK-NEXT: addhn2 v0.8h, v1.4s, v3.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v5.4s +; CHECK-NEXT: addhn2 v0.8h, v1.4s, v5.4s ; CHECK-NEXT: ret entry: %c = sitofp <8 x i32> %a to <8 x bfloat> @@ -7176,15 +7176,15 @@ define <8 x bfloat> @utofp_v8i32_v8bf16(<8 x i32> %a) { ; CHECK-NEXT: ucvtf v0.4s, v0.4s ; CHECK-NEXT: movi v2.4s, #1 ; CHECK-NEXT: ucvtf v1.4s, v1.4s +; CHECK-NEXT: movi v5.4s, #127, msl #8 ; CHECK-NEXT: ushr v3.4s, v0.4s, #16 ; CHECK-NEXT: ushr v4.4s, v1.4s, #16 ; CHECK-NEXT: and v3.16b, v3.16b, v2.16b ; CHECK-NEXT: and v2.16b, v4.16b, v2.16b ; CHECK-NEXT: add v0.4s, v3.4s, v0.4s -; CHECK-NEXT: movi v3.4s, #127, msl #8 ; CHECK-NEXT: add v1.4s, v2.4s, v1.4s -; CHECK-NEXT: addhn v0.4h, v0.4s, v3.4s -; CHECK-NEXT: addhn2 v0.8h, v1.4s, v3.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v5.4s +; CHECK-NEXT: addhn2 v0.8h, v1.4s, v5.4s ; CHECK-NEXT: ret entry: %c = uitofp <8 x i32> %a to <8 x bfloat> @@ -7194,28 +7194,28 @@ entry: define <16 x bfloat> @stofp_v16i32_v16bf16(<16 x i32> %a) { ; CHECK-LABEL: stofp_v16i32_v16bf16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: scvtf v0.4s, v0.4s ; CHECK-NEXT: scvtf v2.4s, v2.4s -; CHECK-NEXT: movi v4.4s, #1 -; CHECK-NEXT: scvtf v1.4s, v1.4s +; CHECK-NEXT: scvtf v0.4s, v0.4s +; CHECK-NEXT: scvtf v4.4s, v1.4s +; CHECK-NEXT: movi v1.4s, #1 ; CHECK-NEXT: scvtf v3.4s, v3.4s +; CHECK-NEXT: movi v17.4s, #127, msl #8 ; CHECK-NEXT: ushr v5.4s, v0.4s, #16 ; CHECK-NEXT: ushr v6.4s, v2.4s, #16 -; CHECK-NEXT: ushr v7.4s, v1.4s, #16 +; CHECK-NEXT: ushr v7.4s, v4.4s, #16 ; CHECK-NEXT: ushr v16.4s, v3.4s, #16 -; CHECK-NEXT: and v5.16b, v5.16b, v4.16b -; CHECK-NEXT: and v6.16b, v6.16b, v4.16b +; CHECK-NEXT: and v5.16b, v5.16b, v1.16b +; CHECK-NEXT: and v6.16b, v6.16b, v1.16b ; CHECK-NEXT: add v0.4s, v5.4s, v0.4s ; CHECK-NEXT: add v2.4s, v6.4s, v2.4s -; CHECK-NEXT: movi v6.4s, #127, msl #8 -; CHECK-NEXT: and v5.16b, v7.16b, v4.16b -; CHECK-NEXT: and v4.16b, v16.16b, v4.16b -; CHECK-NEXT: add v5.4s, v5.4s, v1.4s -; CHECK-NEXT: addhn v0.4h, v0.4s, v6.4s -; CHECK-NEXT: add v3.4s, v4.4s, v3.4s -; CHECK-NEXT: addhn v1.4h, v2.4s, v6.4s -; CHECK-NEXT: addhn2 v0.8h, v5.4s, v6.4s -; CHECK-NEXT: addhn2 v1.8h, v3.4s, v6.4s +; CHECK-NEXT: and v5.16b, v7.16b, v1.16b +; CHECK-NEXT: and v6.16b, v16.16b, v1.16b +; CHECK-NEXT: addhn v0.4h, v0.4s, v17.4s +; CHECK-NEXT: addhn v1.4h, v2.4s, v17.4s +; CHECK-NEXT: add v2.4s, v5.4s, v4.4s +; CHECK-NEXT: add v3.4s, v6.4s, v3.4s +; CHECK-NEXT: addhn2 v0.8h, v2.4s, v17.4s +; CHECK-NEXT: addhn2 v1.8h, v3.4s, v17.4s ; CHECK-NEXT: ret entry: %c = sitofp <16 x i32> %a to <16 x bfloat> @@ -7225,28 +7225,28 @@ entry: define <16 x bfloat> @utofp_v16i32_v16bf16(<16 x i32> %a) { ; CHECK-LABEL: utofp_v16i32_v16bf16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ucvtf v0.4s, v0.4s ; CHECK-NEXT: ucvtf v2.4s, v2.4s -; CHECK-NEXT: movi v4.4s, #1 -; CHECK-NEXT: ucvtf v1.4s, v1.4s +; CHECK-NEXT: ucvtf v0.4s, v0.4s +; CHECK-NEXT: ucvtf v4.4s, v1.4s +; CHECK-NEXT: movi v1.4s, #1 ; CHECK-NEXT: ucvtf v3.4s, v3.4s +; CHECK-NEXT: movi v17.4s, #127, msl #8 ; CHECK-NEXT: ushr v5.4s, v0.4s, #16 ; CHECK-NEXT: ushr v6.4s, v2.4s, #16 -; CHECK-NEXT: ushr v7.4s, v1.4s, #16 +; CHECK-NEXT: ushr v7.4s, v4.4s, #16 ; CHECK-NEXT: ushr v16.4s, v3.4s, #16 -; CHECK-NEXT: and v5.16b, v5.16b, v4.16b -; CHECK-NEXT: and v6.16b, v6.16b, v4.16b +; CHECK-NEXT: and v5.16b, v5.16b, v1.16b +; CHECK-NEXT: and v6.16b, v6.16b, v1.16b ; CHECK-NEXT: add v0.4s, v5.4s, v0.4s ; CHECK-NEXT: add v2.4s, v6.4s, v2.4s -; CHECK-NEXT: movi v6.4s, #127, msl #8 -; CHECK-NEXT: and v5.16b, v7.16b, v4.16b -; CHECK-NEXT: and v4.16b, v16.16b, v4.16b -; CHECK-NEXT: add v5.4s, v5.4s, v1.4s -; CHECK-NEXT: addhn v0.4h, v0.4s, v6.4s -; CHECK-NEXT: add v3.4s, v4.4s, v3.4s -; CHECK-NEXT: addhn v1.4h, v2.4s, v6.4s -; CHECK-NEXT: addhn2 v0.8h, v5.4s, v6.4s -; CHECK-NEXT: addhn2 v1.8h, v3.4s, v6.4s +; CHECK-NEXT: and v5.16b, v7.16b, v1.16b +; CHECK-NEXT: and v6.16b, v16.16b, v1.16b +; CHECK-NEXT: addhn v0.4h, v0.4s, v17.4s +; CHECK-NEXT: addhn v1.4h, v2.4s, v17.4s +; CHECK-NEXT: add v2.4s, v5.4s, v4.4s +; CHECK-NEXT: add v3.4s, v6.4s, v3.4s +; CHECK-NEXT: addhn2 v0.8h, v2.4s, v17.4s +; CHECK-NEXT: addhn2 v1.8h, v3.4s, v17.4s ; CHECK-NEXT: ret entry: %c = uitofp <16 x i32> %a to <16 x bfloat> @@ -7262,42 +7262,42 @@ define <32 x bfloat> @stofp_v32i32_v32bf16(<32 x i32> %a) { ; CHECK-NEXT: scvtf v6.4s, v6.4s ; CHECK-NEXT: movi v16.4s, #1 ; CHECK-NEXT: scvtf v1.4s, v1.4s -; CHECK-NEXT: scvtf v3.4s, v3.4s +; CHECK-NEXT: scvtf v17.4s, v3.4s ; CHECK-NEXT: scvtf v5.4s, v5.4s ; CHECK-NEXT: scvtf v7.4s, v7.4s -; CHECK-NEXT: ushr v17.4s, v0.4s, #16 +; CHECK-NEXT: movi v21.4s, #127, msl #8 +; CHECK-NEXT: ushr v3.4s, v0.4s, #16 ; CHECK-NEXT: ushr v18.4s, v2.4s, #16 ; CHECK-NEXT: ushr v19.4s, v4.4s, #16 ; CHECK-NEXT: ushr v20.4s, v6.4s, #16 -; CHECK-NEXT: ushr v21.4s, v1.4s, #16 -; CHECK-NEXT: ushr v22.4s, v3.4s, #16 -; CHECK-NEXT: ushr v23.4s, v5.4s, #16 -; CHECK-NEXT: and v17.16b, v17.16b, v16.16b +; CHECK-NEXT: ushr v22.4s, v1.4s, #16 +; CHECK-NEXT: ushr v23.4s, v17.4s, #16 +; CHECK-NEXT: ushr v24.4s, v5.4s, #16 +; CHECK-NEXT: ushr v25.4s, v7.4s, #16 +; CHECK-NEXT: and v3.16b, v3.16b, v16.16b ; CHECK-NEXT: and v18.16b, v18.16b, v16.16b ; CHECK-NEXT: and v19.16b, v19.16b, v16.16b ; CHECK-NEXT: and v20.16b, v20.16b, v16.16b -; CHECK-NEXT: and v21.16b, v21.16b, v16.16b -; CHECK-NEXT: and v22.16b, v22.16b, v16.16b -; CHECK-NEXT: add v0.4s, v17.4s, v0.4s -; CHECK-NEXT: ushr v17.4s, v7.4s, #16 +; CHECK-NEXT: add v0.4s, v3.4s, v0.4s +; CHECK-NEXT: and v3.16b, v22.16b, v16.16b ; CHECK-NEXT: add v2.4s, v18.4s, v2.4s -; CHECK-NEXT: movi v18.4s, #127, msl #8 ; CHECK-NEXT: add v4.4s, v19.4s, v4.4s ; CHECK-NEXT: add v6.4s, v20.4s, v6.4s -; CHECK-NEXT: and v19.16b, v23.16b, v16.16b -; CHECK-NEXT: add v20.4s, v22.4s, v3.4s -; CHECK-NEXT: and v16.16b, v17.16b, v16.16b -; CHECK-NEXT: add v17.4s, v21.4s, v1.4s +; CHECK-NEXT: and v18.16b, v23.16b, v16.16b +; CHECK-NEXT: and v19.16b, v24.16b, v16.16b +; CHECK-NEXT: and v16.16b, v25.16b, v16.16b +; CHECK-NEXT: add v20.4s, v3.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v21.4s +; CHECK-NEXT: addhn v1.4h, v2.4s, v21.4s +; CHECK-NEXT: addhn v2.4h, v4.4s, v21.4s +; CHECK-NEXT: addhn v3.4h, v6.4s, v21.4s +; CHECK-NEXT: add v4.4s, v18.4s, v17.4s ; CHECK-NEXT: add v5.4s, v19.4s, v5.4s -; CHECK-NEXT: addhn v0.4h, v0.4s, v18.4s -; CHECK-NEXT: addhn v1.4h, v2.4s, v18.4s -; CHECK-NEXT: addhn v2.4h, v4.4s, v18.4s -; CHECK-NEXT: add v4.4s, v16.4s, v7.4s -; CHECK-NEXT: addhn v3.4h, v6.4s, v18.4s -; CHECK-NEXT: addhn2 v0.8h, v17.4s, v18.4s -; CHECK-NEXT: addhn2 v1.8h, v20.4s, v18.4s -; CHECK-NEXT: addhn2 v2.8h, v5.4s, v18.4s -; CHECK-NEXT: addhn2 v3.8h, v4.4s, v18.4s +; CHECK-NEXT: add v6.4s, v16.4s, v7.4s +; CHECK-NEXT: addhn2 v0.8h, v20.4s, v21.4s +; CHECK-NEXT: addhn2 v1.8h, v4.4s, v21.4s +; CHECK-NEXT: addhn2 v2.8h, v5.4s, v21.4s +; CHECK-NEXT: addhn2 v3.8h, v6.4s, v21.4s ; CHECK-NEXT: ret entry: %c = sitofp <32 x i32> %a to <32 x bfloat> @@ -7313,42 +7313,42 @@ define <32 x bfloat> @utofp_v32i32_v32bf16(<32 x i32> %a) { ; CHECK-NEXT: ucvtf v6.4s, v6.4s ; CHECK-NEXT: movi v16.4s, #1 ; CHECK-NEXT: ucvtf v1.4s, v1.4s -; CHECK-NEXT: ucvtf v3.4s, v3.4s +; CHECK-NEXT: ucvtf v17.4s, v3.4s ; CHECK-NEXT: ucvtf v5.4s, v5.4s ; CHECK-NEXT: ucvtf v7.4s, v7.4s -; CHECK-NEXT: ushr v17.4s, v0.4s, #16 +; CHECK-NEXT: movi v21.4s, #127, msl #8 +; CHECK-NEXT: ushr v3.4s, v0.4s, #16 ; CHECK-NEXT: ushr v18.4s, v2.4s, #16 ; CHECK-NEXT: ushr v19.4s, v4.4s, #16 ; CHECK-NEXT: ushr v20.4s, v6.4s, #16 -; CHECK-NEXT: ushr v21.4s, v1.4s, #16 -; CHECK-NEXT: ushr v22.4s, v3.4s, #16 -; CHECK-NEXT: ushr v23.4s, v5.4s, #16 -; CHECK-NEXT: and v17.16b, v17.16b, v16.16b +; CHECK-NEXT: ushr v22.4s, v1.4s, #16 +; CHECK-NEXT: ushr v23.4s, v17.4s, #16 +; CHECK-NEXT: ushr v24.4s, v5.4s, #16 +; CHECK-NEXT: ushr v25.4s, v7.4s, #16 +; CHECK-NEXT: and v3.16b, v3.16b, v16.16b ; CHECK-NEXT: and v18.16b, v18.16b, v16.16b ; CHECK-NEXT: and v19.16b, v19.16b, v16.16b ; CHECK-NEXT: and v20.16b, v20.16b, v16.16b -; CHECK-NEXT: and v21.16b, v21.16b, v16.16b -; CHECK-NEXT: and v22.16b, v22.16b, v16.16b -; CHECK-NEXT: add v0.4s, v17.4s, v0.4s -; CHECK-NEXT: ushr v17.4s, v7.4s, #16 +; CHECK-NEXT: add v0.4s, v3.4s, v0.4s +; CHECK-NEXT: and v3.16b, v22.16b, v16.16b ; CHECK-NEXT: add v2.4s, v18.4s, v2.4s -; CHECK-NEXT: movi v18.4s, #127, msl #8 ; CHECK-NEXT: add v4.4s, v19.4s, v4.4s ; CHECK-NEXT: add v6.4s, v20.4s, v6.4s -; CHECK-NEXT: and v19.16b, v23.16b, v16.16b -; CHECK-NEXT: add v20.4s, v22.4s, v3.4s -; CHECK-NEXT: and v16.16b, v17.16b, v16.16b -; CHECK-NEXT: add v17.4s, v21.4s, v1.4s +; CHECK-NEXT: and v18.16b, v23.16b, v16.16b +; CHECK-NEXT: and v19.16b, v24.16b, v16.16b +; CHECK-NEXT: and v16.16b, v25.16b, v16.16b +; CHECK-NEXT: add v20.4s, v3.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v21.4s +; CHECK-NEXT: addhn v1.4h, v2.4s, v21.4s +; CHECK-NEXT: addhn v2.4h, v4.4s, v21.4s +; CHECK-NEXT: addhn v3.4h, v6.4s, v21.4s +; CHECK-NEXT: add v4.4s, v18.4s, v17.4s ; CHECK-NEXT: add v5.4s, v19.4s, v5.4s -; CHECK-NEXT: addhn v0.4h, v0.4s, v18.4s -; CHECK-NEXT: addhn v1.4h, v2.4s, v18.4s -; CHECK-NEXT: addhn v2.4h, v4.4s, v18.4s -; CHECK-NEXT: add v4.4s, v16.4s, v7.4s -; CHECK-NEXT: addhn v3.4h, v6.4s, v18.4s -; CHECK-NEXT: addhn2 v0.8h, v17.4s, v18.4s -; CHECK-NEXT: addhn2 v1.8h, v20.4s, v18.4s -; CHECK-NEXT: addhn2 v2.8h, v5.4s, v18.4s -; CHECK-NEXT: addhn2 v3.8h, v4.4s, v18.4s +; CHECK-NEXT: add v6.4s, v16.4s, v7.4s +; CHECK-NEXT: addhn2 v0.8h, v20.4s, v21.4s +; CHECK-NEXT: addhn2 v1.8h, v4.4s, v21.4s +; CHECK-NEXT: addhn2 v2.8h, v5.4s, v21.4s +; CHECK-NEXT: addhn2 v3.8h, v6.4s, v21.4s ; CHECK-NEXT: ret entry: %c = uitofp <32 x i32> %a to <32 x bfloat> @@ -7364,9 +7364,9 @@ define <2 x bfloat> @stofp_v2i16_v2bf16(<2 x i16> %a) { ; CHECK-NEXT: scvtf v0.4s, v0.4s ; CHECK-NEXT: ushr v2.4s, v0.4s, #16 ; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v2.4s ; CHECK-NEXT: ret entry: %c = sitofp <2 x i16> %a to <2 x bfloat> @@ -7382,9 +7382,9 @@ define <2 x bfloat> @utofp_v2i16_v2bf16(<2 x i16> %a) { ; CHECK-NEXT: ucvtf v0.4s, v0.4s ; CHECK-NEXT: ushr v2.4s, v0.4s, #16 ; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v2.4s ; CHECK-NEXT: ret entry: %c = uitofp <2 x i16> %a to <2 x bfloat> @@ -7399,9 +7399,9 @@ define <3 x bfloat> @stofp_v3i16_v3bf16(<3 x i16> %a) { ; CHECK-NEXT: scvtf v0.4s, v0.4s ; CHECK-NEXT: ushr v2.4s, v0.4s, #16 ; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v2.4s ; CHECK-NEXT: ret entry: %c = sitofp <3 x i16> %a to <3 x bfloat> @@ -7416,9 +7416,9 @@ define <3 x bfloat> @utofp_v3i16_v3bf16(<3 x i16> %a) { ; CHECK-NEXT: ucvtf v0.4s, v0.4s ; CHECK-NEXT: ushr v2.4s, v0.4s, #16 ; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v2.4s ; CHECK-NEXT: ret entry: %c = uitofp <3 x i16> %a to <3 x bfloat> @@ -7433,9 +7433,9 @@ define <4 x bfloat> @stofp_v4i16_v4bf16(<4 x i16> %a) { ; CHECK-NEXT: scvtf v0.4s, v0.4s ; CHECK-NEXT: ushr v2.4s, v0.4s, #16 ; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v2.4s ; CHECK-NEXT: ret entry: %c = sitofp <4 x i16> %a to <4 x bfloat> @@ -7450,9 +7450,9 @@ define <4 x bfloat> @utofp_v4i16_v4bf16(<4 x i16> %a) { ; CHECK-NEXT: ucvtf v0.4s, v0.4s ; CHECK-NEXT: ushr v2.4s, v0.4s, #16 ; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v2.4s ; CHECK-NEXT: ret entry: %c = uitofp <4 x i16> %a to <4 x bfloat> @@ -7513,27 +7513,27 @@ define <16 x bfloat> @stofp_v16i16_v16bf16(<16 x i16> %a) { ; CHECK-NEXT: sshll2 v0.4s, v0.8h, #0 ; CHECK-NEXT: sshll2 v1.4s, v1.8h, #0 ; CHECK-NEXT: movi v2.4s, #1 +; CHECK-NEXT: movi v7.4s, #127, msl #8 ; CHECK-NEXT: scvtf v3.4s, v3.4s ; CHECK-NEXT: scvtf v4.4s, v4.4s -; CHECK-NEXT: scvtf v6.4s, v0.4s -; CHECK-NEXT: scvtf v7.4s, v1.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: ushr v5.4s, v3.4s, #16 -; CHECK-NEXT: ushr v0.4s, v4.4s, #16 -; CHECK-NEXT: ushr v16.4s, v6.4s, #16 -; CHECK-NEXT: ushr v17.4s, v7.4s, #16 -; CHECK-NEXT: and v5.16b, v5.16b, v2.16b +; CHECK-NEXT: scvtf v5.4s, v0.4s +; CHECK-NEXT: scvtf v6.4s, v1.4s +; CHECK-NEXT: ushr v0.4s, v3.4s, #16 +; CHECK-NEXT: ushr v1.4s, v4.4s, #16 +; CHECK-NEXT: ushr v16.4s, v5.4s, #16 +; CHECK-NEXT: ushr v17.4s, v6.4s, #16 ; CHECK-NEXT: and v0.16b, v0.16b, v2.16b +; CHECK-NEXT: and v1.16b, v1.16b, v2.16b ; CHECK-NEXT: and v16.16b, v16.16b, v2.16b ; CHECK-NEXT: and v2.16b, v17.16b, v2.16b -; CHECK-NEXT: add v5.4s, v5.4s, v1.4s -; CHECK-NEXT: add v18.4s, v0.4s, v1.4s -; CHECK-NEXT: add v2.4s, v2.4s, v1.4s -; CHECK-NEXT: addhn v0.4h, v3.4s, v5.4s -; CHECK-NEXT: add v3.4s, v16.4s, v1.4s -; CHECK-NEXT: addhn v1.4h, v4.4s, v18.4s -; CHECK-NEXT: addhn2 v0.8h, v6.4s, v3.4s -; CHECK-NEXT: addhn2 v1.8h, v7.4s, v2.4s +; CHECK-NEXT: add v0.4s, v0.4s, v7.4s +; CHECK-NEXT: add v1.4s, v1.4s, v7.4s +; CHECK-NEXT: add v2.4s, v2.4s, v7.4s +; CHECK-NEXT: addhn v0.4h, v3.4s, v0.4s +; CHECK-NEXT: addhn v1.4h, v4.4s, v1.4s +; CHECK-NEXT: add v3.4s, v16.4s, v7.4s +; CHECK-NEXT: addhn2 v0.8h, v5.4s, v3.4s +; CHECK-NEXT: addhn2 v1.8h, v6.4s, v2.4s ; CHECK-NEXT: ret entry: %c = sitofp <16 x i16> %a to <16 x bfloat> @@ -7548,27 +7548,27 @@ define <16 x bfloat> @utofp_v16i16_v16bf16(<16 x i16> %a) { ; CHECK-NEXT: ushll2 v0.4s, v0.8h, #0 ; CHECK-NEXT: ushll2 v1.4s, v1.8h, #0 ; CHECK-NEXT: movi v2.4s, #1 +; CHECK-NEXT: movi v7.4s, #127, msl #8 ; CHECK-NEXT: ucvtf v3.4s, v3.4s ; CHECK-NEXT: ucvtf v4.4s, v4.4s -; CHECK-NEXT: ucvtf v6.4s, v0.4s -; CHECK-NEXT: ucvtf v7.4s, v1.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: ushr v5.4s, v3.4s, #16 -; CHECK-NEXT: ushr v0.4s, v4.4s, #16 -; CHECK-NEXT: ushr v16.4s, v6.4s, #16 -; CHECK-NEXT: ushr v17.4s, v7.4s, #16 -; CHECK-NEXT: and v5.16b, v5.16b, v2.16b +; CHECK-NEXT: ucvtf v5.4s, v0.4s +; CHECK-NEXT: ucvtf v6.4s, v1.4s +; CHECK-NEXT: ushr v0.4s, v3.4s, #16 +; CHECK-NEXT: ushr v1.4s, v4.4s, #16 +; CHECK-NEXT: ushr v16.4s, v5.4s, #16 +; CHECK-NEXT: ushr v17.4s, v6.4s, #16 ; CHECK-NEXT: and v0.16b, v0.16b, v2.16b +; CHECK-NEXT: and v1.16b, v1.16b, v2.16b ; CHECK-NEXT: and v16.16b, v16.16b, v2.16b ; CHECK-NEXT: and v2.16b, v17.16b, v2.16b -; CHECK-NEXT: add v5.4s, v5.4s, v1.4s -; CHECK-NEXT: add v18.4s, v0.4s, v1.4s -; CHECK-NEXT: add v2.4s, v2.4s, v1.4s -; CHECK-NEXT: addhn v0.4h, v3.4s, v5.4s -; CHECK-NEXT: add v3.4s, v16.4s, v1.4s -; CHECK-NEXT: addhn v1.4h, v4.4s, v18.4s -; CHECK-NEXT: addhn2 v0.8h, v6.4s, v3.4s -; CHECK-NEXT: addhn2 v1.8h, v7.4s, v2.4s +; CHECK-NEXT: add v0.4s, v0.4s, v7.4s +; CHECK-NEXT: add v1.4s, v1.4s, v7.4s +; CHECK-NEXT: add v2.4s, v2.4s, v7.4s +; CHECK-NEXT: addhn v0.4h, v3.4s, v0.4s +; CHECK-NEXT: addhn v1.4h, v4.4s, v1.4s +; CHECK-NEXT: add v3.4s, v16.4s, v7.4s +; CHECK-NEXT: addhn2 v0.8h, v5.4s, v3.4s +; CHECK-NEXT: addhn2 v1.8h, v6.4s, v2.4s ; CHECK-NEXT: ret entry: %c = uitofp <16 x i16> %a to <16 x bfloat> @@ -7578,56 +7578,56 @@ entry: define <32 x bfloat> @stofp_v32i16_v32bf16(<32 x i16> %a) { ; CHECK-LABEL: stofp_v32i16_v32bf16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: sshll v5.4s, v1.4h, #0 -; CHECK-NEXT: sshll v4.4s, v0.4h, #0 +; CHECK-NEXT: sshll v4.4s, v1.4h, #0 +; CHECK-NEXT: sshll v5.4s, v0.4h, #0 ; CHECK-NEXT: sshll v6.4s, v2.4h, #0 ; CHECK-NEXT: sshll v7.4s, v3.4h, #0 ; CHECK-NEXT: sshll2 v0.4s, v0.8h, #0 -; CHECK-NEXT: movi v16.4s, #1 ; CHECK-NEXT: sshll2 v1.4s, v1.8h, #0 ; CHECK-NEXT: sshll2 v2.4s, v2.8h, #0 ; CHECK-NEXT: sshll2 v3.4s, v3.8h, #0 +; CHECK-NEXT: movi v16.4s, #1 ; CHECK-NEXT: scvtf v5.4s, v5.4s ; CHECK-NEXT: scvtf v4.4s, v4.4s ; CHECK-NEXT: scvtf v6.4s, v6.4s ; CHECK-NEXT: scvtf v7.4s, v7.4s -; CHECK-NEXT: scvtf v19.4s, v0.4s -; CHECK-NEXT: movi v18.4s, #127, msl #8 -; CHECK-NEXT: scvtf v20.4s, v1.4s -; CHECK-NEXT: scvtf v21.4s, v2.4s -; CHECK-NEXT: scvtf v22.4s, v3.4s +; CHECK-NEXT: scvtf v17.4s, v0.4s +; CHECK-NEXT: scvtf v18.4s, v1.4s +; CHECK-NEXT: scvtf v19.4s, v2.4s +; CHECK-NEXT: scvtf v20.4s, v3.4s +; CHECK-NEXT: movi v21.4s, #127, msl #8 ; CHECK-NEXT: ushr v0.4s, v5.4s, #16 -; CHECK-NEXT: ushr v17.4s, v4.4s, #16 -; CHECK-NEXT: ushr v1.4s, v6.4s, #16 -; CHECK-NEXT: ushr v2.4s, v7.4s, #16 -; CHECK-NEXT: ushr v23.4s, v20.4s, #16 -; CHECK-NEXT: ushr v25.4s, v22.4s, #16 +; CHECK-NEXT: ushr v1.4s, v4.4s, #16 +; CHECK-NEXT: ushr v2.4s, v6.4s, #16 +; CHECK-NEXT: ushr v3.4s, v7.4s, #16 +; CHECK-NEXT: ushr v22.4s, v17.4s, #16 +; CHECK-NEXT: ushr v23.4s, v18.4s, #16 +; CHECK-NEXT: ushr v24.4s, v19.4s, #16 +; CHECK-NEXT: ushr v25.4s, v20.4s, #16 ; CHECK-NEXT: and v0.16b, v0.16b, v16.16b -; CHECK-NEXT: and v3.16b, v17.16b, v16.16b ; CHECK-NEXT: and v1.16b, v1.16b, v16.16b ; CHECK-NEXT: and v2.16b, v2.16b, v16.16b -; CHECK-NEXT: ushr v17.4s, v19.4s, #16 +; CHECK-NEXT: and v3.16b, v3.16b, v16.16b +; CHECK-NEXT: and v22.16b, v22.16b, v16.16b ; CHECK-NEXT: and v23.16b, v23.16b, v16.16b -; CHECK-NEXT: add v24.4s, v0.4s, v18.4s -; CHECK-NEXT: ushr v0.4s, v21.4s, #16 -; CHECK-NEXT: add v3.4s, v3.4s, v18.4s -; CHECK-NEXT: add v26.4s, v1.4s, v18.4s -; CHECK-NEXT: add v27.4s, v2.4s, v18.4s -; CHECK-NEXT: and v17.16b, v17.16b, v16.16b -; CHECK-NEXT: and v28.16b, v0.16b, v16.16b +; CHECK-NEXT: and v24.16b, v24.16b, v16.16b ; CHECK-NEXT: and v16.16b, v25.16b, v16.16b -; CHECK-NEXT: addhn v0.4h, v4.4s, v3.4s -; CHECK-NEXT: addhn v1.4h, v5.4s, v24.4s -; CHECK-NEXT: add v4.4s, v17.4s, v18.4s -; CHECK-NEXT: addhn v2.4h, v6.4s, v26.4s -; CHECK-NEXT: add v5.4s, v23.4s, v18.4s -; CHECK-NEXT: addhn v3.4h, v7.4s, v27.4s -; CHECK-NEXT: add v6.4s, v28.4s, v18.4s -; CHECK-NEXT: add v16.4s, v16.4s, v18.4s -; CHECK-NEXT: addhn2 v0.8h, v19.4s, v4.4s -; CHECK-NEXT: addhn2 v1.8h, v20.4s, v5.4s -; CHECK-NEXT: addhn2 v2.8h, v21.4s, v6.4s -; CHECK-NEXT: addhn2 v3.8h, v22.4s, v16.4s +; CHECK-NEXT: add v0.4s, v0.4s, v21.4s +; CHECK-NEXT: add v1.4s, v1.4s, v21.4s +; CHECK-NEXT: add v2.4s, v2.4s, v21.4s +; CHECK-NEXT: add v3.4s, v3.4s, v21.4s +; CHECK-NEXT: addhn v0.4h, v5.4s, v0.4s +; CHECK-NEXT: addhn v1.4h, v4.4s, v1.4s +; CHECK-NEXT: addhn v2.4h, v6.4s, v2.4s +; CHECK-NEXT: addhn v3.4h, v7.4s, v3.4s +; CHECK-NEXT: add v4.4s, v22.4s, v21.4s +; CHECK-NEXT: add v5.4s, v23.4s, v21.4s +; CHECK-NEXT: add v6.4s, v24.4s, v21.4s +; CHECK-NEXT: add v7.4s, v16.4s, v21.4s +; CHECK-NEXT: addhn2 v0.8h, v17.4s, v4.4s +; CHECK-NEXT: addhn2 v1.8h, v18.4s, v5.4s +; CHECK-NEXT: addhn2 v2.8h, v19.4s, v6.4s +; CHECK-NEXT: addhn2 v3.8h, v20.4s, v7.4s ; CHECK-NEXT: ret entry: %c = sitofp <32 x i16> %a to <32 x bfloat> @@ -7637,56 +7637,56 @@ entry: define <32 x bfloat> @utofp_v32i16_v32bf16(<32 x i16> %a) { ; CHECK-LABEL: utofp_v32i16_v32bf16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ushll v5.4s, v1.4h, #0 -; CHECK-NEXT: ushll v4.4s, v0.4h, #0 +; CHECK-NEXT: ushll v4.4s, v1.4h, #0 +; CHECK-NEXT: ushll v5.4s, v0.4h, #0 ; CHECK-NEXT: ushll v6.4s, v2.4h, #0 ; CHECK-NEXT: ushll v7.4s, v3.4h, #0 ; CHECK-NEXT: ushll2 v0.4s, v0.8h, #0 -; CHECK-NEXT: movi v16.4s, #1 ; CHECK-NEXT: ushll2 v1.4s, v1.8h, #0 ; CHECK-NEXT: ushll2 v2.4s, v2.8h, #0 ; CHECK-NEXT: ushll2 v3.4s, v3.8h, #0 +; CHECK-NEXT: movi v16.4s, #1 ; CHECK-NEXT: ucvtf v5.4s, v5.4s ; CHECK-NEXT: ucvtf v4.4s, v4.4s ; CHECK-NEXT: ucvtf v6.4s, v6.4s ; CHECK-NEXT: ucvtf v7.4s, v7.4s -; CHECK-NEXT: ucvtf v19.4s, v0.4s -; CHECK-NEXT: movi v18.4s, #127, msl #8 -; CHECK-NEXT: ucvtf v20.4s, v1.4s -; CHECK-NEXT: ucvtf v21.4s, v2.4s -; CHECK-NEXT: ucvtf v22.4s, v3.4s +; CHECK-NEXT: ucvtf v17.4s, v0.4s +; CHECK-NEXT: ucvtf v18.4s, v1.4s +; CHECK-NEXT: ucvtf v19.4s, v2.4s +; CHECK-NEXT: ucvtf v20.4s, v3.4s +; CHECK-NEXT: movi v21.4s, #127, msl #8 ; CHECK-NEXT: ushr v0.4s, v5.4s, #16 -; CHECK-NEXT: ushr v17.4s, v4.4s, #16 -; CHECK-NEXT: ushr v1.4s, v6.4s, #16 -; CHECK-NEXT: ushr v2.4s, v7.4s, #16 -; CHECK-NEXT: ushr v23.4s, v20.4s, #16 -; CHECK-NEXT: ushr v25.4s, v22.4s, #16 +; CHECK-NEXT: ushr v1.4s, v4.4s, #16 +; CHECK-NEXT: ushr v2.4s, v6.4s, #16 +; CHECK-NEXT: ushr v3.4s, v7.4s, #16 +; CHECK-NEXT: ushr v22.4s, v17.4s, #16 +; CHECK-NEXT: ushr v23.4s, v18.4s, #16 +; CHECK-NEXT: ushr v24.4s, v19.4s, #16 +; CHECK-NEXT: ushr v25.4s, v20.4s, #16 ; CHECK-NEXT: and v0.16b, v0.16b, v16.16b -; CHECK-NEXT: and v3.16b, v17.16b, v16.16b ; CHECK-NEXT: and v1.16b, v1.16b, v16.16b ; CHECK-NEXT: and v2.16b, v2.16b, v16.16b -; CHECK-NEXT: ushr v17.4s, v19.4s, #16 +; CHECK-NEXT: and v3.16b, v3.16b, v16.16b +; CHECK-NEXT: and v22.16b, v22.16b, v16.16b ; CHECK-NEXT: and v23.16b, v23.16b, v16.16b -; CHECK-NEXT: add v24.4s, v0.4s, v18.4s -; CHECK-NEXT: ushr v0.4s, v21.4s, #16 -; CHECK-NEXT: add v3.4s, v3.4s, v18.4s -; CHECK-NEXT: add v26.4s, v1.4s, v18.4s -; CHECK-NEXT: add v27.4s, v2.4s, v18.4s -; CHECK-NEXT: and v17.16b, v17.16b, v16.16b -; CHECK-NEXT: and v28.16b, v0.16b, v16.16b +; CHECK-NEXT: and v24.16b, v24.16b, v16.16b ; CHECK-NEXT: and v16.16b, v25.16b, v16.16b -; CHECK-NEXT: addhn v0.4h, v4.4s, v3.4s -; CHECK-NEXT: addhn v1.4h, v5.4s, v24.4s -; CHECK-NEXT: add v4.4s, v17.4s, v18.4s -; CHECK-NEXT: addhn v2.4h, v6.4s, v26.4s -; CHECK-NEXT: add v5.4s, v23.4s, v18.4s -; CHECK-NEXT: addhn v3.4h, v7.4s, v27.4s -; CHECK-NEXT: add v6.4s, v28.4s, v18.4s -; CHECK-NEXT: add v16.4s, v16.4s, v18.4s -; CHECK-NEXT: addhn2 v0.8h, v19.4s, v4.4s -; CHECK-NEXT: addhn2 v1.8h, v20.4s, v5.4s -; CHECK-NEXT: addhn2 v2.8h, v21.4s, v6.4s -; CHECK-NEXT: addhn2 v3.8h, v22.4s, v16.4s +; CHECK-NEXT: add v0.4s, v0.4s, v21.4s +; CHECK-NEXT: add v1.4s, v1.4s, v21.4s +; CHECK-NEXT: add v2.4s, v2.4s, v21.4s +; CHECK-NEXT: add v3.4s, v3.4s, v21.4s +; CHECK-NEXT: addhn v0.4h, v5.4s, v0.4s +; CHECK-NEXT: addhn v1.4h, v4.4s, v1.4s +; CHECK-NEXT: addhn v2.4h, v6.4s, v2.4s +; CHECK-NEXT: addhn v3.4h, v7.4s, v3.4s +; CHECK-NEXT: add v4.4s, v22.4s, v21.4s +; CHECK-NEXT: add v5.4s, v23.4s, v21.4s +; CHECK-NEXT: add v6.4s, v24.4s, v21.4s +; CHECK-NEXT: add v7.4s, v16.4s, v21.4s +; CHECK-NEXT: addhn2 v0.8h, v17.4s, v4.4s +; CHECK-NEXT: addhn2 v1.8h, v18.4s, v5.4s +; CHECK-NEXT: addhn2 v2.8h, v19.4s, v6.4s +; CHECK-NEXT: addhn2 v3.8h, v20.4s, v7.4s ; CHECK-NEXT: ret entry: %c = uitofp <32 x i16> %a to <32 x bfloat> @@ -7768,9 +7768,9 @@ define <3 x bfloat> @stofp_v3i8_v3bf16(<3 x i8> %a) { ; CHECK-NEXT: scvtf v0.4s, v0.4s ; CHECK-NEXT: ushr v2.4s, v0.4s, #16 ; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v2.4s ; CHECK-NEXT: ret entry: %c = sitofp <3 x i8> %a to <3 x bfloat> @@ -7789,9 +7789,9 @@ define <3 x bfloat> @utofp_v3i8_v3bf16(<3 x i8> %a) { ; CHECK-NEXT: ucvtf v0.4s, v0.4s ; CHECK-NEXT: ushr v2.4s, v0.4s, #16 ; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v2.4s ; CHECK-NEXT: ret entry: %c = uitofp <3 x i8> %a to <3 x bfloat> @@ -7808,9 +7808,9 @@ define <4 x bfloat> @stofp_v4i8_v4bf16(<4 x i8> %a) { ; CHECK-NEXT: scvtf v0.4s, v0.4s ; CHECK-NEXT: ushr v2.4s, v0.4s, #16 ; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v2.4s ; CHECK-NEXT: ret entry: %c = sitofp <4 x i8> %a to <4 x bfloat> @@ -7826,9 +7826,9 @@ define <4 x bfloat> @utofp_v4i8_v4bf16(<4 x i8> %a) { ; CHECK-NEXT: ucvtf v0.4s, v0.4s ; CHECK-NEXT: ushr v2.4s, v0.4s, #16 ; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v2.4s ; CHECK-NEXT: ret entry: %c = uitofp <4 x i8> %a to <4 x bfloat> @@ -7909,11 +7909,11 @@ define <16 x bfloat> @stofp_v16i8_v16bf16(<16 x i8> %a) { ; CHECK-NEXT: add v5.4s, v5.4s, v7.4s ; CHECK-NEXT: add v0.4s, v0.4s, v7.4s ; CHECK-NEXT: addhn v1.4h, v3.4s, v5.4s -; CHECK-NEXT: add v3.4s, v16.4s, v7.4s -; CHECK-NEXT: add v5.4s, v17.4s, v7.4s ; CHECK-NEXT: addhn v0.4h, v4.4s, v0.4s +; CHECK-NEXT: add v3.4s, v16.4s, v7.4s +; CHECK-NEXT: add v4.4s, v17.4s, v7.4s ; CHECK-NEXT: addhn2 v1.8h, v2.4s, v3.4s -; CHECK-NEXT: addhn2 v0.8h, v6.4s, v5.4s +; CHECK-NEXT: addhn2 v0.8h, v6.4s, v4.4s ; CHECK-NEXT: ret entry: %c = sitofp <16 x i8> %a to <16 x bfloat> @@ -7946,11 +7946,11 @@ define <16 x bfloat> @utofp_v16i8_v16bf16(<16 x i8> %a) { ; CHECK-NEXT: add v5.4s, v5.4s, v7.4s ; CHECK-NEXT: add v0.4s, v0.4s, v7.4s ; CHECK-NEXT: addhn v1.4h, v3.4s, v5.4s -; CHECK-NEXT: add v3.4s, v16.4s, v7.4s -; CHECK-NEXT: add v5.4s, v17.4s, v7.4s ; CHECK-NEXT: addhn v0.4h, v4.4s, v0.4s +; CHECK-NEXT: add v3.4s, v16.4s, v7.4s +; CHECK-NEXT: add v4.4s, v17.4s, v7.4s ; CHECK-NEXT: addhn2 v1.8h, v2.4s, v3.4s -; CHECK-NEXT: addhn2 v0.8h, v6.4s, v5.4s +; CHECK-NEXT: addhn2 v0.8h, v6.4s, v4.4s ; CHECK-NEXT: ret entry: %c = uitofp <16 x i8> %a to <16 x bfloat> @@ -7961,14 +7961,14 @@ define <32 x bfloat> @stofp_v32i8_v32bf16(<32 x i8> %a) { ; CHECK-LABEL: stofp_v32i8_v32bf16: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: sshll2 v3.8h, v0.16b, #0 -; CHECK-NEXT: sshll2 v4.8h, v1.16b, #0 ; CHECK-NEXT: sshll v0.8h, v0.8b, #0 +; CHECK-NEXT: sshll2 v4.8h, v1.16b, #0 ; CHECK-NEXT: sshll v1.8h, v1.8b, #0 ; CHECK-NEXT: movi v2.4s, #1 -; CHECK-NEXT: movi v20.4s, #127, msl #8 +; CHECK-NEXT: movi v21.4s, #127, msl #8 ; CHECK-NEXT: sshll v5.4s, v3.4h, #0 -; CHECK-NEXT: sshll v6.4s, v4.4h, #0 -; CHECK-NEXT: sshll v7.4s, v0.4h, #0 +; CHECK-NEXT: sshll v6.4s, v0.4h, #0 +; CHECK-NEXT: sshll v7.4s, v4.4h, #0 ; CHECK-NEXT: sshll v16.4s, v1.4h, #0 ; CHECK-NEXT: sshll2 v3.4s, v3.8h, #0 ; CHECK-NEXT: sshll2 v4.4s, v4.8h, #0 @@ -7980,40 +7980,40 @@ define <32 x bfloat> @stofp_v32i8_v32bf16(<32 x i8> %a) { ; CHECK-NEXT: scvtf v16.4s, v16.4s ; CHECK-NEXT: scvtf v17.4s, v3.4s ; CHECK-NEXT: scvtf v4.4s, v4.4s -; CHECK-NEXT: scvtf v19.4s, v0.4s -; CHECK-NEXT: scvtf v21.4s, v1.4s -; CHECK-NEXT: ushr v3.4s, v5.4s, #16 -; CHECK-NEXT: ushr v18.4s, v6.4s, #16 -; CHECK-NEXT: ushr v0.4s, v7.4s, #16 -; CHECK-NEXT: ushr v1.4s, v16.4s, #16 -; CHECK-NEXT: ushr v22.4s, v17.4s, #16 -; CHECK-NEXT: ushr v23.4s, v4.4s, #16 -; CHECK-NEXT: ushr v24.4s, v19.4s, #16 -; CHECK-NEXT: ushr v25.4s, v21.4s, #16 -; CHECK-NEXT: and v3.16b, v3.16b, v2.16b -; CHECK-NEXT: and v18.16b, v18.16b, v2.16b +; CHECK-NEXT: scvtf v18.4s, v0.4s +; CHECK-NEXT: scvtf v19.4s, v1.4s +; CHECK-NEXT: ushr v0.4s, v5.4s, #16 +; CHECK-NEXT: ushr v3.4s, v6.4s, #16 +; CHECK-NEXT: ushr v1.4s, v7.4s, #16 +; CHECK-NEXT: ushr v20.4s, v16.4s, #16 +; CHECK-NEXT: ushr v23.4s, v17.4s, #16 +; CHECK-NEXT: ushr v24.4s, v4.4s, #16 +; CHECK-NEXT: ushr v22.4s, v18.4s, #16 +; CHECK-NEXT: ushr v25.4s, v19.4s, #16 ; CHECK-NEXT: and v0.16b, v0.16b, v2.16b +; CHECK-NEXT: and v3.16b, v3.16b, v2.16b ; CHECK-NEXT: and v1.16b, v1.16b, v2.16b -; CHECK-NEXT: and v22.16b, v22.16b, v2.16b +; CHECK-NEXT: and v20.16b, v20.16b, v2.16b ; CHECK-NEXT: and v23.16b, v23.16b, v2.16b ; CHECK-NEXT: and v24.16b, v24.16b, v2.16b -; CHECK-NEXT: and v2.16b, v25.16b, v2.16b -; CHECK-NEXT: add v3.4s, v3.4s, v20.4s -; CHECK-NEXT: add v18.4s, v18.4s, v20.4s -; CHECK-NEXT: add v0.4s, v0.4s, v20.4s -; CHECK-NEXT: add v26.4s, v1.4s, v20.4s -; CHECK-NEXT: addhn v1.4h, v5.4s, v3.4s -; CHECK-NEXT: addhn v3.4h, v6.4s, v18.4s -; CHECK-NEXT: addhn v0.4h, v7.4s, v0.4s -; CHECK-NEXT: add v5.4s, v22.4s, v20.4s -; CHECK-NEXT: add v6.4s, v24.4s, v20.4s -; CHECK-NEXT: add v7.4s, v23.4s, v20.4s -; CHECK-NEXT: add v18.4s, v2.4s, v20.4s -; CHECK-NEXT: addhn v2.4h, v16.4s, v26.4s -; CHECK-NEXT: addhn2 v0.8h, v19.4s, v6.4s -; CHECK-NEXT: addhn2 v1.8h, v17.4s, v5.4s +; CHECK-NEXT: and v22.16b, v22.16b, v2.16b +; CHECK-NEXT: and v25.16b, v25.16b, v2.16b +; CHECK-NEXT: add v0.4s, v0.4s, v21.4s +; CHECK-NEXT: add v3.4s, v3.4s, v21.4s +; CHECK-NEXT: add v26.4s, v1.4s, v21.4s +; CHECK-NEXT: add v20.4s, v20.4s, v21.4s +; CHECK-NEXT: addhn v1.4h, v5.4s, v0.4s +; CHECK-NEXT: addhn v0.4h, v6.4s, v3.4s +; CHECK-NEXT: addhn v3.4h, v7.4s, v26.4s +; CHECK-NEXT: addhn v2.4h, v16.4s, v20.4s +; CHECK-NEXT: add v5.4s, v22.4s, v21.4s +; CHECK-NEXT: add v6.4s, v23.4s, v21.4s +; CHECK-NEXT: add v7.4s, v24.4s, v21.4s +; CHECK-NEXT: add v16.4s, v25.4s, v21.4s +; CHECK-NEXT: addhn2 v0.8h, v18.4s, v5.4s +; CHECK-NEXT: addhn2 v1.8h, v17.4s, v6.4s ; CHECK-NEXT: addhn2 v3.8h, v4.4s, v7.4s -; CHECK-NEXT: addhn2 v2.8h, v21.4s, v18.4s +; CHECK-NEXT: addhn2 v2.8h, v19.4s, v16.4s ; CHECK-NEXT: ret entry: %c = sitofp <32 x i8> %a to <32 x bfloat> @@ -8024,14 +8024,14 @@ define <32 x bfloat> @utofp_v32i8_v32bf16(<32 x i8> %a) { ; CHECK-LABEL: utofp_v32i8_v32bf16: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: ushll2 v3.8h, v0.16b, #0 -; CHECK-NEXT: ushll2 v4.8h, v1.16b, #0 ; CHECK-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-NEXT: ushll2 v4.8h, v1.16b, #0 ; CHECK-NEXT: ushll v1.8h, v1.8b, #0 ; CHECK-NEXT: movi v2.4s, #1 -; CHECK-NEXT: movi v20.4s, #127, msl #8 +; CHECK-NEXT: movi v21.4s, #127, msl #8 ; CHECK-NEXT: ushll v5.4s, v3.4h, #0 -; CHECK-NEXT: ushll v6.4s, v4.4h, #0 -; CHECK-NEXT: ushll v7.4s, v0.4h, #0 +; CHECK-NEXT: ushll v6.4s, v0.4h, #0 +; CHECK-NEXT: ushll v7.4s, v4.4h, #0 ; CHECK-NEXT: ushll v16.4s, v1.4h, #0 ; CHECK-NEXT: ushll2 v3.4s, v3.8h, #0 ; CHECK-NEXT: ushll2 v4.4s, v4.8h, #0 @@ -8043,40 +8043,40 @@ define <32 x bfloat> @utofp_v32i8_v32bf16(<32 x i8> %a) { ; CHECK-NEXT: ucvtf v16.4s, v16.4s ; CHECK-NEXT: ucvtf v17.4s, v3.4s ; CHECK-NEXT: ucvtf v4.4s, v4.4s -; CHECK-NEXT: ucvtf v19.4s, v0.4s -; CHECK-NEXT: ucvtf v21.4s, v1.4s -; CHECK-NEXT: ushr v3.4s, v5.4s, #16 -; CHECK-NEXT: ushr v18.4s, v6.4s, #16 -; CHECK-NEXT: ushr v0.4s, v7.4s, #16 -; CHECK-NEXT: ushr v1.4s, v16.4s, #16 -; CHECK-NEXT: ushr v22.4s, v17.4s, #16 -; CHECK-NEXT: ushr v23.4s, v4.4s, #16 -; CHECK-NEXT: ushr v24.4s, v19.4s, #16 -; CHECK-NEXT: ushr v25.4s, v21.4s, #16 -; CHECK-NEXT: and v3.16b, v3.16b, v2.16b -; CHECK-NEXT: and v18.16b, v18.16b, v2.16b +; CHECK-NEXT: ucvtf v18.4s, v0.4s +; CHECK-NEXT: ucvtf v19.4s, v1.4s +; CHECK-NEXT: ushr v0.4s, v5.4s, #16 +; CHECK-NEXT: ushr v3.4s, v6.4s, #16 +; CHECK-NEXT: ushr v1.4s, v7.4s, #16 +; CHECK-NEXT: ushr v20.4s, v16.4s, #16 +; CHECK-NEXT: ushr v23.4s, v17.4s, #16 +; CHECK-NEXT: ushr v24.4s, v4.4s, #16 +; CHECK-NEXT: ushr v22.4s, v18.4s, #16 +; CHECK-NEXT: ushr v25.4s, v19.4s, #16 ; CHECK-NEXT: and v0.16b, v0.16b, v2.16b +; CHECK-NEXT: and v3.16b, v3.16b, v2.16b ; CHECK-NEXT: and v1.16b, v1.16b, v2.16b -; CHECK-NEXT: and v22.16b, v22.16b, v2.16b +; CHECK-NEXT: and v20.16b, v20.16b, v2.16b ; CHECK-NEXT: and v23.16b, v23.16b, v2.16b ; CHECK-NEXT: and v24.16b, v24.16b, v2.16b -; CHECK-NEXT: and v2.16b, v25.16b, v2.16b -; CHECK-NEXT: add v3.4s, v3.4s, v20.4s -; CHECK-NEXT: add v18.4s, v18.4s, v20.4s -; CHECK-NEXT: add v0.4s, v0.4s, v20.4s -; CHECK-NEXT: add v26.4s, v1.4s, v20.4s -; CHECK-NEXT: addhn v1.4h, v5.4s, v3.4s -; CHECK-NEXT: addhn v3.4h, v6.4s, v18.4s -; CHECK-NEXT: addhn v0.4h, v7.4s, v0.4s -; CHECK-NEXT: add v5.4s, v22.4s, v20.4s -; CHECK-NEXT: add v6.4s, v24.4s, v20.4s -; CHECK-NEXT: add v7.4s, v23.4s, v20.4s -; CHECK-NEXT: add v18.4s, v2.4s, v20.4s -; CHECK-NEXT: addhn v2.4h, v16.4s, v26.4s -; CHECK-NEXT: addhn2 v0.8h, v19.4s, v6.4s -; CHECK-NEXT: addhn2 v1.8h, v17.4s, v5.4s +; CHECK-NEXT: and v22.16b, v22.16b, v2.16b +; CHECK-NEXT: and v25.16b, v25.16b, v2.16b +; CHECK-NEXT: add v0.4s, v0.4s, v21.4s +; CHECK-NEXT: add v3.4s, v3.4s, v21.4s +; CHECK-NEXT: add v26.4s, v1.4s, v21.4s +; CHECK-NEXT: add v20.4s, v20.4s, v21.4s +; CHECK-NEXT: addhn v1.4h, v5.4s, v0.4s +; CHECK-NEXT: addhn v0.4h, v6.4s, v3.4s +; CHECK-NEXT: addhn v3.4h, v7.4s, v26.4s +; CHECK-NEXT: addhn v2.4h, v16.4s, v20.4s +; CHECK-NEXT: add v5.4s, v22.4s, v21.4s +; CHECK-NEXT: add v6.4s, v23.4s, v21.4s +; CHECK-NEXT: add v7.4s, v24.4s, v21.4s +; CHECK-NEXT: add v16.4s, v25.4s, v21.4s +; CHECK-NEXT: addhn2 v0.8h, v18.4s, v5.4s +; CHECK-NEXT: addhn2 v1.8h, v17.4s, v6.4s ; CHECK-NEXT: addhn2 v3.8h, v4.4s, v7.4s -; CHECK-NEXT: addhn2 v2.8h, v21.4s, v18.4s +; CHECK-NEXT: addhn2 v2.8h, v19.4s, v16.4s ; CHECK-NEXT: ret entry: %c = uitofp <32 x i8> %a to <32 x bfloat> diff --git a/llvm/test/CodeGen/AArch64/ldexp.ll b/llvm/test/CodeGen/AArch64/ldexp.ll index 4b491051a88a..ba04ba1d7bb6 100644 --- a/llvm/test/CodeGen/AArch64/ldexp.ll +++ b/llvm/test/CodeGen/AArch64/ldexp.ll @@ -4,9 +4,9 @@ define double @testExp(double %val, i32 %a) { ; CHECK-LABEL: testExp: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 ; CHECK-NEXT: sxtw x8, w0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: fmov d1, x8 ; CHECK-NEXT: fscale z0.d, p0/m, z0.d, z1.d @@ -22,8 +22,8 @@ declare double @ldexp(double, i32) memory(none) define float @testExpf(float %val, i32 %a) { ; CHECK-LABEL: testExpf: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: fmov s1, w0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: // kill: def $s0 killed $s0 def $z0 ; CHECK-NEXT: fscale z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: // kill: def $s0 killed $s0 killed $z0 @@ -49,9 +49,9 @@ declare fp128 @ldexpl(fp128, i32) memory(none) define half @testExpf16(half %val, i32 %a) { ; CHECK-LABEL: testExpf16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: fcvt s0, h0 ; CHECK-NEXT: fmov s1, w0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: fscale z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: fcvt h0, s0 ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/llvm-ir-to-intrinsic.ll b/llvm/test/CodeGen/AArch64/llvm-ir-to-intrinsic.ll index ab15bf564ec4..59a460923e8b 100644 --- a/llvm/test/CodeGen/AArch64/llvm-ir-to-intrinsic.ll +++ b/llvm/test/CodeGen/AArch64/llvm-ir-to-intrinsic.ll @@ -10,9 +10,9 @@ define @sdiv_i8( %a, %b) ; CHECK: // %bb.0: ; CHECK-NEXT: sunpkhi z2.h, z1.b ; CHECK-NEXT: sunpkhi z3.h, z0.b -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sunpklo z1.h, z1.b ; CHECK-NEXT: sunpklo z0.h, z0.b +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sunpkhi z4.s, z2.h ; CHECK-NEXT: sunpkhi z5.s, z3.h ; CHECK-NEXT: sunpklo z2.s, z2.h @@ -36,11 +36,11 @@ define @sdiv_i8( %a, %b) define @sdiv_i16( %a, %b) { ; CHECK-LABEL: sdiv_i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sunpkhi z2.s, z1.h ; CHECK-NEXT: sunpkhi z3.s, z0.h ; CHECK-NEXT: sunpklo z1.s, z1.h ; CHECK-NEXT: sunpklo z0.s, z0.h +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sdivr z2.s, p0/m, z2.s, z3.s ; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: uzp1 z0.h, z0.h, z2.h @@ -140,9 +140,9 @@ define @srem_i8( %a, %b) define @srem_i16( %a, %b) { ; CHECK-LABEL: srem_i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sunpkhi z2.s, z1.h ; CHECK-NEXT: sunpkhi z3.s, z0.h +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sunpklo z4.s, z0.h ; CHECK-NEXT: sdivr z2.s, p0/m, z2.s, z3.s ; CHECK-NEXT: sunpklo z3.s, z1.h @@ -188,9 +188,9 @@ define @udiv_i8( %a, %b) ; CHECK: // %bb.0: ; CHECK-NEXT: uunpkhi z2.h, z1.b ; CHECK-NEXT: uunpkhi z3.h, z0.b -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uunpklo z1.h, z1.b ; CHECK-NEXT: uunpklo z0.h, z0.b +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uunpkhi z4.s, z2.h ; CHECK-NEXT: uunpkhi z5.s, z3.h ; CHECK-NEXT: uunpklo z2.s, z2.h @@ -214,11 +214,11 @@ define @udiv_i8( %a, %b) define @udiv_i16( %a, %b) { ; CHECK-LABEL: udiv_i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uunpkhi z2.s, z1.h ; CHECK-NEXT: uunpkhi z3.s, z0.h ; CHECK-NEXT: uunpklo z1.s, z1.h ; CHECK-NEXT: uunpklo z0.s, z0.h +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: udivr z2.s, p0/m, z2.s, z3.s ; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: uzp1 z0.h, z0.h, z2.h @@ -261,9 +261,9 @@ define @udiv_split_i32( %a, @udiv_widen_i32( %a, %b) { ; CHECK-LABEL: udiv_widen_i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and z1.d, z1.d, #0xffffffff ; CHECK-NEXT: and z0.d, z0.d, #0xffffffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: udiv z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret %div = udiv %a, %b @@ -319,9 +319,9 @@ define @urem_i8( %a, %b) define @urem_i16( %a, %b) { ; CHECK-LABEL: urem_i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uunpkhi z2.s, z1.h ; CHECK-NEXT: uunpkhi z3.s, z0.h +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uunpklo z4.s, z0.h ; CHECK-NEXT: udivr z2.s, p0/m, z2.s, z3.s ; CHECK-NEXT: uunpklo z3.s, z1.h @@ -558,9 +558,9 @@ define @umin_split_i64( %a, @umin_promote_i8( %a, %b) { ; CHECK-LABEL: umin_promote_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: and z1.h, z1.h, #0xff ; CHECK-NEXT: and z0.h, z0.h, #0xff +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: umin z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret %cmp = icmp ult %a, %b @@ -704,9 +704,9 @@ define @umax_split_i16( %a, @umax_promote_i32( %a, %b) { ; CHECK-LABEL: umax_promote_i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and z1.d, z1.d, #0xffffffff ; CHECK-NEXT: and z0.d, z0.d, #0xffffffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: umax z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret %cmp = icmp ugt %a, %b @@ -883,8 +883,8 @@ define @lsl_split_i64( %a, @lsl_promote_i16( %a, %b){ ; CHECK-LABEL: lsl_promote_i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and z1.s, z1.s, #0xffff +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: lsl z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %shl = shl %a, %b @@ -982,9 +982,9 @@ define @lsr_i64( %a, %b) define @lsr_promote_i8( %a, %b){ ; CHECK-LABEL: lsr_promote_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: and z1.h, z1.h, #0xff ; CHECK-NEXT: and z0.h, z0.h, #0xff +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: lsr z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret %shr = lshr %a, %b @@ -1081,10 +1081,10 @@ declare @llvm.fshr.nxv2i64(, @fshl_i64( %a, %b, %c){ ; CHECK-LABEL: fshl_i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z3.d, #63 // =0x3f ; CHECK-NEXT: mov z4.d, z2.d ; CHECK-NEXT: lsr z1.d, z1.d, #1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: bic z2.d, z3.d, z2.d ; CHECK-NEXT: and z4.d, z4.d, #0x3f ; CHECK-NEXT: lsl z0.d, p0/m, z0.d, z4.d @@ -1098,17 +1098,16 @@ define @fshl_i64( %a, %b define @fshl_illegal_i64( %a, %b, %c){ ; CHECK-LABEL: fshl_illegal_i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z6.d, #63 // =0x3f -; CHECK-NEXT: mov z7.d, z4.d ; CHECK-NEXT: lsr z2.d, z2.d, #1 ; CHECK-NEXT: lsr z3.d, z3.d, #1 -; CHECK-NEXT: bic z4.d, z6.d, z4.d -; CHECK-NEXT: and z7.d, z7.d, #0x3f +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: bic z7.d, z6.d, z4.d +; CHECK-NEXT: and z4.d, z4.d, #0x3f ; CHECK-NEXT: bic z6.d, z6.d, z5.d ; CHECK-NEXT: and z5.d, z5.d, #0x3f -; CHECK-NEXT: lsl z0.d, p0/m, z0.d, z7.d -; CHECK-NEXT: lsr z2.d, p0/m, z2.d, z4.d +; CHECK-NEXT: lsl z0.d, p0/m, z0.d, z4.d +; CHECK-NEXT: lsr z2.d, p0/m, z2.d, z7.d ; CHECK-NEXT: lsr z3.d, p0/m, z3.d, z6.d ; CHECK-NEXT: lsl z1.d, p0/m, z1.d, z5.d ; CHECK-NEXT: orr z0.d, z0.d, z2.d @@ -1121,9 +1120,9 @@ define @fshl_illegal_i64( %a, @fshl_rot_i64( %a, %b){ ; CHECK-LABEL: fshl_rot_i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: subr z1.d, z1.d, #0 // =0x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and z2.d, z2.d, #0x3f ; CHECK-NEXT: and z1.d, z1.d, #0x3f ; CHECK-NEXT: lslr z2.d, p0/m, z2.d, z0.d @@ -1138,11 +1137,11 @@ define @fshl_rot_i64( %a, @fshl_rot_illegal_i64( %a, %b){ ; CHECK-LABEL: fshl_rot_illegal_i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z4.d, z2.d ; CHECK-NEXT: subr z2.d, z2.d, #0 // =0x0 ; CHECK-NEXT: mov z5.d, z3.d ; CHECK-NEXT: subr z3.d, z3.d, #0 // =0x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and z4.d, z4.d, #0x3f ; CHECK-NEXT: and z2.d, z2.d, #0x3f ; CHECK-NEXT: and z5.d, z5.d, #0x3f @@ -1175,10 +1174,10 @@ define @fshl_rot_const_i64( %a){ define @fshr_i64( %a, %b, %c){ ; CHECK-LABEL: fshr_i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z3.d, #63 // =0x3f ; CHECK-NEXT: mov z4.d, z2.d ; CHECK-NEXT: lsl z0.d, z0.d, #1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: bic z2.d, z3.d, z2.d ; CHECK-NEXT: and z4.d, z4.d, #0x3f ; CHECK-NEXT: lsr z1.d, p0/m, z1.d, z4.d @@ -1192,9 +1191,9 @@ define @fshr_i64( %a, %b define @fshr_rot_i64( %a, %b){ ; CHECK-LABEL: fshr_rot_i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: subr z1.d, z1.d, #0 // =0x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and z2.d, z2.d, #0x3f ; CHECK-NEXT: and z1.d, z1.d, #0x3f ; CHECK-NEXT: lsrr z2.d, p0/m, z2.d, z0.d diff --git a/llvm/test/CodeGen/AArch64/load-insert-zero.ll b/llvm/test/CodeGen/AArch64/load-insert-zero.ll index 993af08a66dd..23d545459295 100644 --- a/llvm/test/CodeGen/AArch64/load-insert-zero.ll +++ b/llvm/test/CodeGen/AArch64/load-insert-zero.ll @@ -469,18 +469,18 @@ define void @predictor_4x4_neon(ptr nocapture noundef writeonly %0, i64 noundef ; CHECK-NEXT: lsr w8, w8, #24 ; CHECK-NEXT: uaddl v0.8h, v0.8b, v1.8b ; CHECK-NEXT: urhadd v1.8b, v1.8b, v2.8b -; CHECK-NEXT: str s1, [x0] ; CHECK-NEXT: add v0.8h, v0.8h, v3.8h ; CHECK-NEXT: dup v3.8b, w8 +; CHECK-NEXT: str s1, [x0] ; CHECK-NEXT: lsl x8, x1, #1 ; CHECK-NEXT: rshrn v0.8b, v0.8h, #2 ; CHECK-NEXT: zip1 v2.2s, v1.2s, v3.2s ; CHECK-NEXT: str s0, [x0, x1] ; CHECK-NEXT: zip1 v3.2s, v0.2s, v3.2s ; CHECK-NEXT: ext v2.8b, v2.8b, v0.8b, #1 +; CHECK-NEXT: ext v1.8b, v3.8b, v0.8b, #1 ; CHECK-NEXT: str s2, [x0, x8] ; CHECK-NEXT: add x8, x8, x1 -; CHECK-NEXT: ext v1.8b, v3.8b, v0.8b, #1 ; CHECK-NEXT: str s1, [x0, x8] ; CHECK-NEXT: ret %5 = load i32, ptr %2, align 4 @@ -608,9 +608,9 @@ define void @predictor_4x4_neon_new(ptr nocapture noundef writeonly %0, i64 noun define @loadnxv8i8(ptr %p) { ; CHECK-LABEL: loadnxv8i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl1 ; CHECK-NEXT: mov z0.h, #0 // =0x0 ; CHECK-NEXT: ldrb w8, [x0] +; CHECK-NEXT: ptrue p0.h, vl1 ; CHECK-NEXT: mov z0.h, p0/m, w8 ; CHECK-NEXT: ret %l = load i8, ptr %p @@ -631,9 +631,9 @@ define @loadnxv16i8(ptr %p) { define @loadnxv4i16(ptr %p) { ; CHECK-LABEL: loadnxv4i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl1 ; CHECK-NEXT: mov z0.s, #0 // =0x0 ; CHECK-NEXT: ldrh w8, [x0] +; CHECK-NEXT: ptrue p0.s, vl1 ; CHECK-NEXT: mov z0.s, p0/m, w8 ; CHECK-NEXT: ret %l = load i16, ptr %p @@ -654,9 +654,9 @@ define @loadnxv8i16(ptr %p) { define @loadnxv2i32(ptr %p) { ; CHECK-LABEL: loadnxv2i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl1 ; CHECK-NEXT: mov z0.d, #0 // =0x0 ; CHECK-NEXT: ldr w8, [x0] +; CHECK-NEXT: ptrue p0.d, vl1 ; CHECK-NEXT: mov z0.d, p0/m, x8 ; CHECK-NEXT: ret %l = load i32, ptr %p @@ -688,9 +688,9 @@ define @loadnxv2i64(ptr %p) { define @loadnxv4f16(ptr %p) { ; CHECK-LABEL: loadnxv4f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, wzr ; CHECK-NEXT: index z0.s, #0, #1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, z1.s ; CHECK-NEXT: mov z0.h, #0 // =0x0 @@ -715,9 +715,9 @@ define @loadnxv8f16(ptr %p) { define @loadnxv4bf16(ptr %p) { ; CHECK-LABEL: loadnxv4bf16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, wzr ; CHECK-NEXT: index z0.s, #0, #1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, z1.s ; CHECK-NEXT: mov z0.h, #0 // =0x0 @@ -742,9 +742,9 @@ define @loadnxv8bf16(ptr %p) { define @loadnxv2f32(ptr %p) { ; CHECK-LABEL: loadnxv2f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, xzr ; CHECK-NEXT: index z0.d, #0, #1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: cmpeq p0.d, p0/z, z0.d, z1.d ; CHECK-NEXT: mov z0.s, #0 // =0x0 @@ -782,9 +782,9 @@ define @loadnxv2f64(ptr %p) { define @loadnxv8i8_offset(ptr %p) { ; CHECK-LABEL: loadnxv8i8_offset: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl1 ; CHECK-NEXT: mov z0.h, #0 // =0x0 ; CHECK-NEXT: ldrb w8, [x0, #1] +; CHECK-NEXT: ptrue p0.h, vl1 ; CHECK-NEXT: mov z0.h, p0/m, w8 ; CHECK-NEXT: ret %g = getelementptr inbounds i8, ptr %p, i64 1 @@ -807,9 +807,9 @@ define @loadnxv16i8_offset(ptr %p) { define @loadnxv4i16_offset(ptr %p) { ; CHECK-LABEL: loadnxv4i16_offset: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl1 ; CHECK-NEXT: mov z0.s, #0 // =0x0 ; CHECK-NEXT: ldurh w8, [x0, #1] +; CHECK-NEXT: ptrue p0.s, vl1 ; CHECK-NEXT: mov z0.s, p0/m, w8 ; CHECK-NEXT: ret %g = getelementptr inbounds i8, ptr %p, i64 1 @@ -832,9 +832,9 @@ define @loadnxv8i16_offset(ptr %p) { define @loadnxv2i32_offset(ptr %p) { ; CHECK-LABEL: loadnxv2i32_offset: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl1 ; CHECK-NEXT: mov z0.d, #0 // =0x0 ; CHECK-NEXT: ldur w8, [x0, #1] +; CHECK-NEXT: ptrue p0.d, vl1 ; CHECK-NEXT: mov z0.d, p0/m, x8 ; CHECK-NEXT: ret %g = getelementptr inbounds i8, ptr %p, i64 1 @@ -869,9 +869,9 @@ define @loadnxv2i64_offset(ptr %p) { define @loadnxv4f16_offset(ptr %p) { ; CHECK-LABEL: loadnxv4f16_offset: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, wzr ; CHECK-NEXT: index z0.s, #0, #1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, z1.s ; CHECK-NEXT: mov z0.h, #0 // =0x0 @@ -898,9 +898,9 @@ define @loadnxv8f16_offset(ptr %p) { define @loadnxv4bf16_offset(ptr %p) { ; CHECK-LABEL: loadnxv4bf16_offset: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, wzr ; CHECK-NEXT: index z0.s, #0, #1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, z1.s ; CHECK-NEXT: mov z0.h, #0 // =0x0 @@ -927,9 +927,9 @@ define @loadnxv8bf16_offset(ptr %p) { define @loadnxv2f32_offset(ptr %p) { ; CHECK-LABEL: loadnxv2f32_offset: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, xzr ; CHECK-NEXT: index z0.d, #0, #1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: cmpeq p0.d, p0/z, z0.d, z1.d ; CHECK-NEXT: mov z0.s, #0 // =0x0 diff --git a/llvm/test/CodeGen/AArch64/logic-shift.ll b/llvm/test/CodeGen/AArch64/logic-shift.ll index 39f82dd4593f..31047954401c 100644 --- a/llvm/test/CodeGen/AArch64/logic-shift.ll +++ b/llvm/test/CodeGen/AArch64/logic-shift.ll @@ -34,9 +34,9 @@ define i32 @or_lshr_commute1(i32 %x0, i32 %x1, i32 %y, i32 %z) { define <8 x i16> @or_lshr_commute2(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %y, <8 x i16> %z) { ; CHECK-LABEL: or_lshr_commute2: ; CHECK: // %bb.0: -; CHECK-NEXT: neg v2.8h, v2.8h ; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b -; CHECK-NEXT: ushl v0.8h, v0.8h, v2.8h +; CHECK-NEXT: neg v1.8h, v2.8h +; CHECK-NEXT: ushl v0.8h, v0.8h, v1.8h ; CHECK-NEXT: orr v0.16b, v0.16b, v3.16b ; CHECK-NEXT: ret %sh1 = lshr <8 x i16> %x0, %y @@ -49,9 +49,9 @@ define <8 x i16> @or_lshr_commute2(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %y, < define <2 x i64> @or_lshr_commute3(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %y, <2 x i64> %z) { ; CHECK-LABEL: or_lshr_commute3: ; CHECK: // %bb.0: -; CHECK-NEXT: neg v2.2d, v2.2d ; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b -; CHECK-NEXT: ushl v0.2d, v0.2d, v2.2d +; CHECK-NEXT: neg v1.2d, v2.2d +; CHECK-NEXT: ushl v0.2d, v0.2d, v1.2d ; CHECK-NEXT: orr v0.16b, v0.16b, v3.16b ; CHECK-NEXT: ret %sh1 = lshr <2 x i64> %x0, %y @@ -94,9 +94,9 @@ define i64 @or_ashr_commute1(i64 %x0, i64 %x1, i64 %y, i64 %z) { define <4 x i32> @or_ashr_commute2(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %y, <4 x i32> %z) { ; CHECK-LABEL: or_ashr_commute2: ; CHECK: // %bb.0: -; CHECK-NEXT: neg v2.4s, v2.4s ; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b -; CHECK-NEXT: sshl v0.4s, v0.4s, v2.4s +; CHECK-NEXT: neg v1.4s, v2.4s +; CHECK-NEXT: sshl v0.4s, v0.4s, v1.4s ; CHECK-NEXT: orr v0.16b, v0.16b, v3.16b ; CHECK-NEXT: ret %sh1 = ashr <4 x i32> %x0, %y @@ -109,9 +109,9 @@ define <4 x i32> @or_ashr_commute2(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %y, < define <16 x i8> @or_ashr_commute3(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %y, <16 x i8> %z) { ; CHECK-LABEL: or_ashr_commute3: ; CHECK: // %bb.0: -; CHECK-NEXT: neg v2.16b, v2.16b ; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b -; CHECK-NEXT: sshl v0.16b, v0.16b, v2.16b +; CHECK-NEXT: neg v1.16b, v2.16b +; CHECK-NEXT: sshl v0.16b, v0.16b, v1.16b ; CHECK-NEXT: orr v0.16b, v0.16b, v3.16b ; CHECK-NEXT: ret %sh1 = ashr <16 x i8> %x0, %y @@ -262,9 +262,9 @@ define i32 @xor_lshr_commute1(i32 %x0, i32 %x1, i32 %y, i32 %z) { define <8 x i16> @xor_lshr_commute2(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %y, <8 x i16> %z) { ; CHECK-LABEL: xor_lshr_commute2: ; CHECK: // %bb.0: -; CHECK-NEXT: neg v2.8h, v2.8h ; CHECK-NEXT: eor v0.16b, v0.16b, v1.16b -; CHECK-NEXT: ushl v0.8h, v0.8h, v2.8h +; CHECK-NEXT: neg v1.8h, v2.8h +; CHECK-NEXT: ushl v0.8h, v0.8h, v1.8h ; CHECK-NEXT: eor v0.16b, v0.16b, v3.16b ; CHECK-NEXT: ret %sh1 = lshr <8 x i16> %x0, %y @@ -277,9 +277,9 @@ define <8 x i16> @xor_lshr_commute2(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %y, define <2 x i64> @xor_lshr_commute3(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %y, <2 x i64> %z) { ; CHECK-LABEL: xor_lshr_commute3: ; CHECK: // %bb.0: -; CHECK-NEXT: neg v2.2d, v2.2d ; CHECK-NEXT: eor v0.16b, v0.16b, v1.16b -; CHECK-NEXT: ushl v0.2d, v0.2d, v2.2d +; CHECK-NEXT: neg v1.2d, v2.2d +; CHECK-NEXT: ushl v0.2d, v0.2d, v1.2d ; CHECK-NEXT: eor v0.16b, v0.16b, v3.16b ; CHECK-NEXT: ret %sh1 = lshr <2 x i64> %x0, %y @@ -322,9 +322,9 @@ define i64 @xor_ashr_commute1(i64 %x0, i64 %x1, i64 %y, i64 %z) { define <4 x i32> @xor_ashr_commute2(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %y, <4 x i32> %z) { ; CHECK-LABEL: xor_ashr_commute2: ; CHECK: // %bb.0: -; CHECK-NEXT: neg v2.4s, v2.4s ; CHECK-NEXT: eor v0.16b, v0.16b, v1.16b -; CHECK-NEXT: sshl v0.4s, v0.4s, v2.4s +; CHECK-NEXT: neg v1.4s, v2.4s +; CHECK-NEXT: sshl v0.4s, v0.4s, v1.4s ; CHECK-NEXT: eor v0.16b, v0.16b, v3.16b ; CHECK-NEXT: ret %sh1 = ashr <4 x i32> %x0, %y @@ -337,9 +337,9 @@ define <4 x i32> @xor_ashr_commute2(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %y, define <16 x i8> @xor_ashr_commute3(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %y, <16 x i8> %z) { ; CHECK-LABEL: xor_ashr_commute3: ; CHECK: // %bb.0: -; CHECK-NEXT: neg v2.16b, v2.16b ; CHECK-NEXT: eor v0.16b, v0.16b, v1.16b -; CHECK-NEXT: sshl v0.16b, v0.16b, v2.16b +; CHECK-NEXT: neg v1.16b, v2.16b +; CHECK-NEXT: sshl v0.16b, v0.16b, v1.16b ; CHECK-NEXT: eor v0.16b, v0.16b, v3.16b ; CHECK-NEXT: ret %sh1 = ashr <16 x i8> %x0, %y @@ -490,9 +490,9 @@ define i32 @and_lshr_commute1(i32 %x0, i32 %x1, i32 %y, i32 %z) { define <8 x i16> @and_lshr_commute2(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %y, <8 x i16> %z) { ; CHECK-LABEL: and_lshr_commute2: ; CHECK: // %bb.0: -; CHECK-NEXT: neg v2.8h, v2.8h ; CHECK-NEXT: and v0.16b, v0.16b, v1.16b -; CHECK-NEXT: ushl v0.8h, v0.8h, v2.8h +; CHECK-NEXT: neg v1.8h, v2.8h +; CHECK-NEXT: ushl v0.8h, v0.8h, v1.8h ; CHECK-NEXT: and v0.16b, v0.16b, v3.16b ; CHECK-NEXT: ret %sh1 = lshr <8 x i16> %x0, %y @@ -505,9 +505,9 @@ define <8 x i16> @and_lshr_commute2(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %y, define <2 x i64> @and_lshr_commute3(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %y, <2 x i64> %z) { ; CHECK-LABEL: and_lshr_commute3: ; CHECK: // %bb.0: -; CHECK-NEXT: neg v2.2d, v2.2d ; CHECK-NEXT: and v0.16b, v0.16b, v1.16b -; CHECK-NEXT: ushl v0.2d, v0.2d, v2.2d +; CHECK-NEXT: neg v1.2d, v2.2d +; CHECK-NEXT: ushl v0.2d, v0.2d, v1.2d ; CHECK-NEXT: and v0.16b, v0.16b, v3.16b ; CHECK-NEXT: ret %sh1 = lshr <2 x i64> %x0, %y @@ -550,9 +550,9 @@ define i64 @and_ashr_commute1(i64 %x0, i64 %x1, i64 %y, i64 %z) { define <4 x i32> @and_ashr_commute2(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %y, <4 x i32> %z) { ; CHECK-LABEL: and_ashr_commute2: ; CHECK: // %bb.0: -; CHECK-NEXT: neg v2.4s, v2.4s ; CHECK-NEXT: and v0.16b, v0.16b, v1.16b -; CHECK-NEXT: sshl v0.4s, v0.4s, v2.4s +; CHECK-NEXT: neg v1.4s, v2.4s +; CHECK-NEXT: sshl v0.4s, v0.4s, v1.4s ; CHECK-NEXT: and v0.16b, v0.16b, v3.16b ; CHECK-NEXT: ret %sh1 = ashr <4 x i32> %x0, %y @@ -565,9 +565,9 @@ define <4 x i32> @and_ashr_commute2(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %y, define <16 x i8> @and_ashr_commute3(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %y, <16 x i8> %z) { ; CHECK-LABEL: and_ashr_commute3: ; CHECK: // %bb.0: -; CHECK-NEXT: neg v2.16b, v2.16b ; CHECK-NEXT: and v0.16b, v0.16b, v1.16b -; CHECK-NEXT: sshl v0.16b, v0.16b, v2.16b +; CHECK-NEXT: neg v1.16b, v2.16b +; CHECK-NEXT: sshl v0.16b, v0.16b, v1.16b ; CHECK-NEXT: and v0.16b, v0.16b, v3.16b ; CHECK-NEXT: ret %sh1 = ashr <16 x i8> %x0, %y diff --git a/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll b/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll index 06570b4539cc..fac96e07de54 100644 --- a/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll +++ b/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll @@ -258,10 +258,10 @@ define @splice_nxv2i1_idx( %a, @llvm.experimental.vector.splice.nxv2i1( %a, %b, i32 1) ret %res @@ -273,10 +273,10 @@ define @splice_nxv4i1_idx( %a, @llvm.experimental.vector.splice.nxv4i1( %a, %b, i32 2) ret %res @@ -288,10 +288,10 @@ define @splice_nxv8i1_idx( %a, @llvm.experimental.vector.splice.nxv8i1( %a, %b, i32 4) ret %res @@ -303,10 +303,10 @@ define @splice_nxv16i1_idx( %a, @llvm.experimental.vector.splice.nxv16i1( %a, %b, i32 8) ret %res @@ -350,16 +350,16 @@ define @splice_nxv16f32_16( %a, @splice_nxv16i8_neg17( %a, @splice_nxv8i16_neg9( %a, @splice_nxv8f16_neg9( %a, @splice_nxv2i1( %a, ; CHECK-NEXT: ptrue p2.d, vl1 ; CHECK-NEXT: mov z0.d, p1/z, #1 // =0x1 ; CHECK-NEXT: mov z1.d, p0/z, #1 // =0x1 +; CHECK-NEXT: rev p0.d, p2.d +; CHECK-NEXT: splice z1.d, p0, z1.d, z0.d ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: rev p2.d, p2.d -; CHECK-NEXT: splice z1.d, p2, z1.d, z0.d ; CHECK-NEXT: and z1.d, z1.d, #0x1 ; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0 ; CHECK-NEXT: ret @@ -716,9 +716,9 @@ define @splice_nxv4i1( %a, ; CHECK-NEXT: ptrue p2.s, vl1 ; CHECK-NEXT: mov z0.s, p1/z, #1 // =0x1 ; CHECK-NEXT: mov z1.s, p0/z, #1 // =0x1 +; CHECK-NEXT: rev p0.s, p2.s +; CHECK-NEXT: splice z1.s, p0, z1.s, z0.s ; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: rev p2.s, p2.s -; CHECK-NEXT: splice z1.s, p2, z1.s, z0.s ; CHECK-NEXT: and z1.s, z1.s, #0x1 ; CHECK-NEXT: cmpne p0.s, p0/z, z1.s, #0 ; CHECK-NEXT: ret @@ -733,9 +733,9 @@ define @splice_nxv8i1( %a, ; CHECK-NEXT: ptrue p2.h, vl1 ; CHECK-NEXT: mov z0.h, p1/z, #1 // =0x1 ; CHECK-NEXT: mov z1.h, p0/z, #1 // =0x1 +; CHECK-NEXT: rev p0.h, p2.h +; CHECK-NEXT: splice z1.h, p0, z1.h, z0.h ; CHECK-NEXT: ptrue p0.h -; CHECK-NEXT: rev p2.h, p2.h -; CHECK-NEXT: splice z1.h, p2, z1.h, z0.h ; CHECK-NEXT: and z1.h, z1.h, #0x1 ; CHECK-NEXT: cmpne p0.h, p0/z, z1.h, #0 ; CHECK-NEXT: ret @@ -750,9 +750,9 @@ define @splice_nxv16i1( %a, @splice_nxv8i32( %a, @splice_nxv16f32_neg17( %a, %a, <24 x i8> %b, <24 x i8> %c, <24 ; CHECK-NEXT: ld1 { v5.b }[15], [x8] ; CHECK-NEXT: ld1 { v7.b }[7], [x10] ; CHECK-NEXT: addp v1.2s, v19.2s, v19.2s +; CHECK-NEXT: fmov w8, s0 ; CHECK-NEXT: sdot v16.4s, v5.16b, v4.16b ; CHECK-NEXT: sdot v18.2s, v7.8b, v6.8b -; CHECK-NEXT: fmov w8, s0 ; CHECK-NEXT: fmov w9, s1 ; CHECK-NEXT: addv s2, v16.4s ; CHECK-NEXT: addp v3.2s, v18.2s, v18.2s @@ -975,8 +975,8 @@ define i32 @test_sdot_v24i8_double_nomla(<24 x i8> %a, <24 x i8> %b, <24 x i8> % ; CHECK-NEXT: addv s3, v5.4s ; CHECK-NEXT: addp v1.2s, v17.2s, v17.2s ; CHECK-NEXT: addp v2.2s, v7.2s, v7.2s -; CHECK-NEXT: addv s0, v6.4s ; CHECK-NEXT: fmov w10, s3 +; CHECK-NEXT: addv s0, v6.4s ; CHECK-NEXT: fmov w9, s1 ; CHECK-NEXT: fmov w11, s2 ; CHECK-NEXT: fmov w8, s0 @@ -998,26 +998,26 @@ entry: define i32 @test_udot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { ; CHECK-LABEL: test_udot_v25i8: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ldp q4, q0, [x0] +; CHECK-NEXT: ldp q2, q0, [x0] ; CHECK-NEXT: ldp q5, q1, [x1] -; CHECK-NEXT: ushll2 v2.8h, v0.16b, #0 -; CHECK-NEXT: ushll v6.8h, v4.8b, #0 +; CHECK-NEXT: ushll2 v3.8h, v0.16b, #0 +; CHECK-NEXT: ushll v6.8h, v2.8b, #0 ; CHECK-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-NEXT: ushll2 v3.8h, v1.16b, #0 +; CHECK-NEXT: ushll2 v4.8h, v1.16b, #0 ; CHECK-NEXT: ushll v7.8h, v5.8b, #0 ; CHECK-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-NEXT: umull v2.4s, v3.4h, v2.4h -; CHECK-NEXT: movi v3.2d, #0000000000000000 +; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0 +; CHECK-NEXT: umull v3.4s, v4.4h, v3.4h +; CHECK-NEXT: movi v4.2d, #0000000000000000 ; CHECK-NEXT: umull2 v16.4s, v7.8h, v6.8h ; CHECK-NEXT: umull v6.4s, v7.4h, v6.4h -; CHECK-NEXT: mov v3.s[0], v2.s[0] -; CHECK-NEXT: ushll2 v2.8h, v4.16b, #0 -; CHECK-NEXT: ushll2 v4.8h, v5.16b, #0 -; CHECK-NEXT: umlal v6.4s, v1.4h, v0.4h +; CHECK-NEXT: mov v4.s[0], v3.s[0] +; CHECK-NEXT: ushll2 v3.8h, v5.16b, #0 ; CHECK-NEXT: umlal2 v16.4s, v1.8h, v0.8h -; CHECK-NEXT: umlal v3.4s, v4.4h, v2.4h -; CHECK-NEXT: umlal2 v16.4s, v4.8h, v2.8h -; CHECK-NEXT: add v0.4s, v6.4s, v3.4s +; CHECK-NEXT: umlal v6.4s, v1.4h, v0.4h +; CHECK-NEXT: umlal v4.4s, v3.4h, v2.4h +; CHECK-NEXT: umlal2 v16.4s, v3.8h, v2.8h +; CHECK-NEXT: add v0.4s, v6.4s, v4.4s ; CHECK-NEXT: add v0.4s, v0.4s, v16.4s ; CHECK-NEXT: addv s0, v0.4s ; CHECK-NEXT: fmov w8, s0 @@ -1039,17 +1039,17 @@ define i32 @test_udot_v25i8_nomla(ptr nocapture readonly %a1) { ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: ldp q2, q1, [x0] ; CHECK-NEXT: movi v0.2d, #0000000000000000 -; CHECK-NEXT: ushll v3.8h, v1.8b, #0 +; CHECK-NEXT: ushll2 v3.8h, v1.16b, #0 +; CHECK-NEXT: ushll v1.8h, v1.8b, #0 ; CHECK-NEXT: ushll v4.8h, v2.8b, #0 -; CHECK-NEXT: ushll2 v1.8h, v1.16b, #0 ; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0 -; CHECK-NEXT: uaddl2 v5.4s, v4.8h, v3.8h -; CHECK-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-NEXT: uaddl v3.4s, v4.4h, v3.4h -; CHECK-NEXT: mov v0.s[0], v1.s[0] -; CHECK-NEXT: uaddw2 v1.4s, v5.4s, v2.8h +; CHECK-NEXT: ushll v3.4s, v3.4h, #0 +; CHECK-NEXT: uaddl2 v5.4s, v4.8h, v1.8h +; CHECK-NEXT: uaddl v1.4s, v4.4h, v1.4h +; CHECK-NEXT: mov v0.s[0], v3.s[0] +; CHECK-NEXT: uaddw2 v3.4s, v5.4s, v2.8h +; CHECK-NEXT: add v1.4s, v1.4s, v3.4s ; CHECK-NEXT: uaddw v0.4s, v0.4s, v2.4h -; CHECK-NEXT: add v1.4s, v3.4s, v1.4s ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s ; CHECK-NEXT: addv s0, v0.4s ; CHECK-NEXT: fmov w0, s0 @@ -1063,26 +1063,26 @@ entry: define i32 @test_sdot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { ; CHECK-LABEL: test_sdot_v25i8: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ldp q4, q0, [x0] +; CHECK-NEXT: ldp q2, q0, [x0] ; CHECK-NEXT: ldp q5, q1, [x1] -; CHECK-NEXT: sshll2 v2.8h, v0.16b, #0 -; CHECK-NEXT: sshll v6.8h, v4.8b, #0 +; CHECK-NEXT: sshll2 v3.8h, v0.16b, #0 +; CHECK-NEXT: sshll v6.8h, v2.8b, #0 ; CHECK-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-NEXT: sshll2 v3.8h, v1.16b, #0 +; CHECK-NEXT: sshll2 v4.8h, v1.16b, #0 ; CHECK-NEXT: sshll v7.8h, v5.8b, #0 ; CHECK-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-NEXT: smull v2.4s, v3.4h, v2.4h -; CHECK-NEXT: movi v3.2d, #0000000000000000 +; CHECK-NEXT: sshll2 v2.8h, v2.16b, #0 +; CHECK-NEXT: smull v3.4s, v4.4h, v3.4h +; CHECK-NEXT: movi v4.2d, #0000000000000000 ; CHECK-NEXT: smull2 v16.4s, v7.8h, v6.8h ; CHECK-NEXT: smull v6.4s, v7.4h, v6.4h -; CHECK-NEXT: mov v3.s[0], v2.s[0] -; CHECK-NEXT: sshll2 v2.8h, v4.16b, #0 -; CHECK-NEXT: sshll2 v4.8h, v5.16b, #0 -; CHECK-NEXT: smlal v6.4s, v1.4h, v0.4h +; CHECK-NEXT: mov v4.s[0], v3.s[0] +; CHECK-NEXT: sshll2 v3.8h, v5.16b, #0 ; CHECK-NEXT: smlal2 v16.4s, v1.8h, v0.8h -; CHECK-NEXT: smlal v3.4s, v4.4h, v2.4h -; CHECK-NEXT: smlal2 v16.4s, v4.8h, v2.8h -; CHECK-NEXT: add v0.4s, v6.4s, v3.4s +; CHECK-NEXT: smlal v6.4s, v1.4h, v0.4h +; CHECK-NEXT: smlal v4.4s, v3.4h, v2.4h +; CHECK-NEXT: smlal2 v16.4s, v3.8h, v2.8h +; CHECK-NEXT: add v0.4s, v6.4s, v4.4s ; CHECK-NEXT: add v0.4s, v0.4s, v16.4s ; CHECK-NEXT: addv s0, v0.4s ; CHECK-NEXT: fmov w8, s0 @@ -1105,222 +1105,222 @@ define i32 @test_sdot_v25i8_double(<25 x i8> %a, <25 x i8> %b, <25 x i8> %c, <25 ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: .cfi_def_cfa_offset 16 ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: fmov s4, w0 -; CHECK-NEXT: ldr b0, [sp, #80] -; CHECK-NEXT: add x8, sp, #88 ; CHECK-NEXT: ldr b1, [sp, #16] -; CHECK-NEXT: add x10, sp, #24 -; CHECK-NEXT: ldr b2, [sp, #280] -; CHECK-NEXT: ld1 { v0.b }[1], [x8] +; CHECK-NEXT: ldr b0, [sp, #80] +; CHECK-NEXT: add x11, sp, #24 ; CHECK-NEXT: ldr b3, [sp, #216] +; CHECK-NEXT: add x10, sp, #88 +; CHECK-NEXT: ldr b2, [sp, #280] +; CHECK-NEXT: ld1 { v1.b }[1], [x11] ; CHECK-NEXT: add x11, sp, #224 -; CHECK-NEXT: mov v4.b[1], w1 -; CHECK-NEXT: ld1 { v1.b }[1], [x10] +; CHECK-NEXT: ldr b4, [sp, #152] +; CHECK-NEXT: ldr b6, [sp, #480] +; CHECK-NEXT: ld1 { v0.b }[1], [x10] ; CHECK-NEXT: add x10, sp, #288 -; CHECK-NEXT: ldr b5, [sp, #152] -; CHECK-NEXT: add x9, sp, #96 -; CHECK-NEXT: ld1 { v2.b }[1], [x10] +; CHECK-NEXT: add x12, sp, #160 ; CHECK-NEXT: ld1 { v3.b }[1], [x11] -; CHECK-NEXT: add x10, sp, #160 -; CHECK-NEXT: ld1 { v0.b }[2], [x9] -; CHECK-NEXT: ld1 { v5.b }[1], [x10] -; CHECK-NEXT: add x10, sp, #32 -; CHECK-NEXT: add x11, sp, #296 -; CHECK-NEXT: mov v4.b[2], w2 -; CHECK-NEXT: ld1 { v1.b }[2], [x10] -; CHECK-NEXT: add x10, sp, #232 +; CHECK-NEXT: add x11, sp, #488 +; CHECK-NEXT: ld1 { v2.b }[1], [x10] +; CHECK-NEXT: ld1 { v4.b }[1], [x12] +; CHECK-NEXT: ld1 { v6.b }[1], [x11] +; CHECK-NEXT: add x11, sp, #32 +; CHECK-NEXT: add x9, sp, #96 ; CHECK-NEXT: add x8, sp, #104 -; CHECK-NEXT: ld1 { v2.b }[2], [x11] -; CHECK-NEXT: ld1 { v3.b }[2], [x10] +; CHECK-NEXT: ld1 { v1.b }[2], [x11] +; CHECK-NEXT: add x11, sp, #232 +; CHECK-NEXT: ld1 { v0.b }[2], [x9] +; CHECK-NEXT: add x9, sp, #296 +; CHECK-NEXT: ld1 { v3.b }[2], [x11] ; CHECK-NEXT: add x11, sp, #168 +; CHECK-NEXT: ld1 { v2.b }[2], [x9] +; CHECK-NEXT: ld1 { v4.b }[2], [x11] +; CHECK-NEXT: add x11, sp, #40 +; CHECK-NEXT: ld1 { v1.b }[3], [x11] ; CHECK-NEXT: ld1 { v0.b }[3], [x8] -; CHECK-NEXT: add x8, sp, #40 -; CHECK-NEXT: ld1 { v5.b }[2], [x11] -; CHECK-NEXT: ld1 { v1.b }[3], [x8] -; CHECK-NEXT: add x8, sp, #240 -; CHECK-NEXT: mov v4.b[3], w3 -; CHECK-NEXT: ld1 { v3.b }[3], [x8] -; CHECK-NEXT: add x8, sp, #176 -; CHECK-NEXT: add x12, sp, #112 -; CHECK-NEXT: add x13, sp, #48 -; CHECK-NEXT: add x9, sp, #120 -; CHECK-NEXT: ld1 { v5.b }[3], [x8] -; CHECK-NEXT: ld1 { v0.b }[4], [x12] -; CHECK-NEXT: add x12, sp, #184 -; CHECK-NEXT: ld1 { v1.b }[4], [x13] -; CHECK-NEXT: add x15, sp, #56 -; CHECK-NEXT: add x14, sp, #128 -; CHECK-NEXT: mov v4.b[4], w4 -; CHECK-NEXT: add x11, sp, #304 -; CHECK-NEXT: add x13, sp, #256 -; CHECK-NEXT: ld1 { v5.b }[4], [x12] -; CHECK-NEXT: ld1 { v0.b }[5], [x9] -; CHECK-NEXT: add x9, sp, #192 -; CHECK-NEXT: add x12, sp, #248 -; CHECK-NEXT: ld1 { v1.b }[5], [x15] -; CHECK-NEXT: add x15, sp, #200 -; CHECK-NEXT: ld1 { v3.b }[4], [x12] -; CHECK-NEXT: ld1 { v2.b }[3], [x11] +; CHECK-NEXT: add x8, sp, #304 +; CHECK-NEXT: add x10, sp, #112 +; CHECK-NEXT: add x11, sp, #240 +; CHECK-NEXT: add x13, sp, #56 +; CHECK-NEXT: ld1 { v2.b }[3], [x8] +; CHECK-NEXT: add x8, sp, #48 +; CHECK-NEXT: ld1 { v3.b }[3], [x11] +; CHECK-NEXT: ld1 { v1.b }[4], [x8] +; CHECK-NEXT: ld1 { v0.b }[4], [x10] +; CHECK-NEXT: add x15, sp, #312 +; CHECK-NEXT: add x12, sp, #120 +; CHECK-NEXT: add x8, sp, #248 ; CHECK-NEXT: add x11, sp, #64 -; CHECK-NEXT: mov v4.b[5], w5 -; CHECK-NEXT: ld1 { v5.b }[5], [x9] -; CHECK-NEXT: ld1 { v0.b }[6], [x14] -; CHECK-NEXT: ldr b6, [sp, #352] -; CHECK-NEXT: add x10, sp, #136 -; CHECK-NEXT: ld1 { v1.b }[6], [x11] -; CHECK-NEXT: add x11, sp, #360 -; CHECK-NEXT: ld1 { v3.b }[5], [x13] +; CHECK-NEXT: ld1 { v2.b }[4], [x15] +; CHECK-NEXT: ld1 { v3.b }[4], [x8] +; CHECK-NEXT: add x15, sp, #320 +; CHECK-NEXT: ld1 { v1.b }[5], [x13] +; CHECK-NEXT: ld1 { v0.b }[5], [x12] ; CHECK-NEXT: ldr b18, [sp, #552] -; CHECK-NEXT: ld1 { v5.b }[6], [x15] -; CHECK-NEXT: add x14, sp, #208 -; CHECK-NEXT: ld1 { v6.b }[1], [x11] -; CHECK-NEXT: mov v4.b[6], w6 -; CHECK-NEXT: ld1 { v0.b }[7], [x10] -; CHECK-NEXT: add x10, sp, #560 -; CHECK-NEXT: add x9, sp, #264 -; CHECK-NEXT: ld1 { v18.b }[1], [x10] +; CHECK-NEXT: add x14, sp, #128 +; CHECK-NEXT: add x16, sp, #256 +; CHECK-NEXT: ldr b16, [sp, #352] +; CHECK-NEXT: ld1 { v2.b }[5], [x15] +; CHECK-NEXT: add x15, sp, #176 +; CHECK-NEXT: ld1 { v3.b }[5], [x16] +; CHECK-NEXT: ld1 { v1.b }[6], [x11] +; CHECK-NEXT: add x11, sp, #560 +; CHECK-NEXT: ld1 { v0.b }[6], [x14] +; CHECK-NEXT: add x16, sp, #360 +; CHECK-NEXT: ld1 { v4.b }[3], [x15] +; CHECK-NEXT: ld1 { v18.b }[1], [x11] +; CHECK-NEXT: add x10, sp, #72 +; CHECK-NEXT: ld1 { v16.b }[1], [x16] +; CHECK-NEXT: add x9, sp, #136 +; CHECK-NEXT: add x14, sp, #184 +; CHECK-NEXT: ld1 { v1.b }[7], [x10] ; CHECK-NEXT: add x10, sp, #568 -; CHECK-NEXT: ld1 { v5.b }[7], [x14] -; CHECK-NEXT: ld1 { v3.b }[6], [x9] +; CHECK-NEXT: ld1 { v0.b }[7], [x9] +; CHECK-NEXT: ld1 { v4.b }[4], [x14] ; CHECK-NEXT: add x9, sp, #368 -; CHECK-NEXT: ld1 { v6.b }[2], [x9] -; CHECK-NEXT: add x11, sp, #488 -; CHECK-NEXT: ldr b7, [sp, #144] -; CHECK-NEXT: mov v4.b[7], w7 ; CHECK-NEXT: ld1 { v18.b }[2], [x10] -; CHECK-NEXT: add x10, sp, #376 -; CHECK-NEXT: sshll v17.8h, v5.8b, #0 -; CHECK-NEXT: ldr b5, [sp, #480] -; CHECK-NEXT: sshll v7.8h, v7.8b, #0 -; CHECK-NEXT: ld1 { v6.b }[3], [x10] +; CHECK-NEXT: add x11, sp, #496 +; CHECK-NEXT: ld1 { v16.b }[2], [x9] +; CHECK-NEXT: fmov s5, w0 +; CHECK-NEXT: add x9, sp, #192 +; CHECK-NEXT: ld1 { v6.b }[2], [x11] ; CHECK-NEXT: add x10, sp, #576 -; CHECK-NEXT: add x8, sp, #312 -; CHECK-NEXT: ld1 { v5.b }[1], [x11] +; CHECK-NEXT: ld1 { v4.b }[5], [x9] +; CHECK-NEXT: add x9, sp, #376 ; CHECK-NEXT: ld1 { v18.b }[3], [x10] -; CHECK-NEXT: add x11, sp, #496 -; CHECK-NEXT: sshll v16.8h, v4.8b, #0 -; CHECK-NEXT: ldr b4, [sp, #344] -; CHECK-NEXT: add x10, sp, #384 -; CHECK-NEXT: ld1 { v6.b }[4], [x10] +; CHECK-NEXT: add x11, sp, #504 +; CHECK-NEXT: ld1 { v16.b }[3], [x9] +; CHECK-NEXT: mov v5.b[1], w1 +; CHECK-NEXT: ldr b7, [sp, #144] +; CHECK-NEXT: ldr b17, [sp, #344] +; CHECK-NEXT: add x9, sp, #200 +; CHECK-NEXT: ld1 { v6.b }[3], [x11] ; CHECK-NEXT: add x10, sp, #584 -; CHECK-NEXT: ld1 { v2.b }[4], [x8] -; CHECK-NEXT: sshll v19.8h, v4.8b, #0 -; CHECK-NEXT: ld1 { v5.b }[2], [x11] +; CHECK-NEXT: ld1 { v4.b }[6], [x9] +; CHECK-NEXT: add x9, sp, #384 ; CHECK-NEXT: ld1 { v18.b }[4], [x10] -; CHECK-NEXT: smull2 v4.4s, v16.8h, v17.8h -; CHECK-NEXT: smull v16.4s, v16.4h, v17.4h -; CHECK-NEXT: ldr b17, [sp, #416] -; CHECK-NEXT: add x11, sp, #504 -; CHECK-NEXT: add x10, sp, #424 -; CHECK-NEXT: add x16, sp, #320 -; CHECK-NEXT: smull v19.4s, v7.4h, v19.4h -; CHECK-NEXT: movi v7.2d, #0000000000000000 -; CHECK-NEXT: ld1 { v5.b }[3], [x11] -; CHECK-NEXT: add x11, sp, #392 -; CHECK-NEXT: ld1 { v17.b }[1], [x10] -; CHECK-NEXT: add x10, sp, #592 -; CHECK-NEXT: ld1 { v2.b }[5], [x16] -; CHECK-NEXT: ld1 { v6.b }[5], [x11] -; CHECK-NEXT: ld1 { v18.b }[5], [x10] +; CHECK-NEXT: sshll v7.8h, v7.8b, #0 +; CHECK-NEXT: sshll v17.8h, v17.8b, #0 ; CHECK-NEXT: add x11, sp, #512 -; CHECK-NEXT: add x10, sp, #432 +; CHECK-NEXT: ld1 { v16.b }[4], [x9] +; CHECK-NEXT: ld1 { v6.b }[4], [x11] +; CHECK-NEXT: add x11, sp, #592 +; CHECK-NEXT: mov v5.b[2], w2 +; CHECK-NEXT: add x10, sp, #392 +; CHECK-NEXT: ldr b19, [sp, #680] +; CHECK-NEXT: ld1 { v18.b }[5], [x11] +; CHECK-NEXT: smull v7.4s, v7.4h, v17.4h +; CHECK-NEXT: ldr b17, [sp, #416] +; CHECK-NEXT: ld1 { v16.b }[5], [x10] +; CHECK-NEXT: add x10, sp, #688 ; CHECK-NEXT: add x12, sp, #328 -; CHECK-NEXT: mov v7.s[0], v19.s[0] -; CHECK-NEXT: ld1 { v5.b }[4], [x11] -; CHECK-NEXT: add x11, sp, #400 -; CHECK-NEXT: ld1 { v17.b }[2], [x10] +; CHECK-NEXT: add x9, sp, #424 +; CHECK-NEXT: ld1 { v19.b }[1], [x10] ; CHECK-NEXT: add x10, sp, #600 -; CHECK-NEXT: ldr b19, [sp, #680] ; CHECK-NEXT: ldr b20, [sp, #616] ; CHECK-NEXT: ld1 { v2.b }[6], [x12] -; CHECK-NEXT: ld1 { v6.b }[6], [x11] +; CHECK-NEXT: ld1 { v17.b }[1], [x9] +; CHECK-NEXT: add x11, sp, #400 ; CHECK-NEXT: ld1 { v18.b }[6], [x10] -; CHECK-NEXT: add x11, sp, #688 ; CHECK-NEXT: add x12, sp, #624 -; CHECK-NEXT: ld1 { v19.b }[1], [x11] +; CHECK-NEXT: mov v5.b[3], w3 +; CHECK-NEXT: ld1 { v16.b }[6], [x11] +; CHECK-NEXT: add x11, sp, #696 ; CHECK-NEXT: ld1 { v20.b }[1], [x12] -; CHECK-NEXT: add x10, sp, #408 +; CHECK-NEXT: add x9, sp, #432 +; CHECK-NEXT: ld1 { v19.b }[2], [x11] ; CHECK-NEXT: add x11, sp, #608 -; CHECK-NEXT: add x12, sp, #440 -; CHECK-NEXT: ld1 { v6.b }[7], [x10] +; CHECK-NEXT: ld1 { v17.b }[2], [x9] +; CHECK-NEXT: add x10, sp, #408 ; CHECK-NEXT: ld1 { v18.b }[7], [x11] -; CHECK-NEXT: ld1 { v17.b }[3], [x12] -; CHECK-NEXT: add x10, sp, #696 ; CHECK-NEXT: add x11, sp, #632 -; CHECK-NEXT: ld1 { v19.b }[2], [x10] -; CHECK-NEXT: add x10, sp, #448 +; CHECK-NEXT: ld1 { v16.b }[7], [x10] ; CHECK-NEXT: ld1 { v20.b }[2], [x11] -; CHECK-NEXT: add x11, sp, #640 -; CHECK-NEXT: sshll v6.8h, v6.8b, #0 -; CHECK-NEXT: ld1 { v17.b }[4], [x10] +; CHECK-NEXT: mov v5.b[4], w4 ; CHECK-NEXT: add x10, sp, #704 -; CHECK-NEXT: sshll v18.8h, v18.8b, #0 +; CHECK-NEXT: add x12, sp, #440 ; CHECK-NEXT: ld1 { v19.b }[3], [x10] -; CHECK-NEXT: add x10, sp, #712 -; CHECK-NEXT: add x12, sp, #520 -; CHECK-NEXT: ld1 { v20.b }[3], [x11] -; CHECK-NEXT: add x11, sp, #648 -; CHECK-NEXT: ldr b21, [sp, #544] -; CHECK-NEXT: smull2 v22.4s, v6.8h, v18.8h -; CHECK-NEXT: smull v6.4s, v6.4h, v18.4h -; CHECK-NEXT: ldr b18, [sp, #744] -; CHECK-NEXT: ld1 { v19.b }[4], [x10] -; CHECK-NEXT: ld1 { v5.b }[5], [x12] -; CHECK-NEXT: add x12, sp, #656 -; CHECK-NEXT: ld1 { v20.b }[4], [x11] -; CHECK-NEXT: add x11, sp, #456 -; CHECK-NEXT: sshll v21.8h, v21.8b, #0 -; CHECK-NEXT: ld1 { v17.b }[5], [x11] -; CHECK-NEXT: add x11, sp, #720 +; CHECK-NEXT: add x10, sp, #448 +; CHECK-NEXT: ld1 { v17.b }[3], [x12] +; CHECK-NEXT: add x12, sp, #640 +; CHECK-NEXT: sshll v21.8h, v16.8b, #0 +; CHECK-NEXT: ld1 { v20.b }[3], [x12] ; CHECK-NEXT: sshll v18.8h, v18.8b, #0 +; CHECK-NEXT: add x11, sp, #712 +; CHECK-NEXT: mov v5.b[5], w5 +; CHECK-NEXT: ld1 { v19.b }[4], [x11] +; CHECK-NEXT: add x9, sp, #520 +; CHECK-NEXT: ld1 { v17.b }[4], [x10] +; CHECK-NEXT: add x10, sp, #648 +; CHECK-NEXT: ldr b22, [sp, #544] +; CHECK-NEXT: ld1 { v20.b }[4], [x10] +; CHECK-NEXT: smull2 v16.4s, v21.8h, v18.8h +; CHECK-NEXT: smull v18.4s, v21.4h, v18.4h +; CHECK-NEXT: ldr b21, [sp, #744] +; CHECK-NEXT: add x11, sp, #720 +; CHECK-NEXT: ld1 { v6.b }[5], [x9] +; CHECK-NEXT: add x9, sp, #456 ; CHECK-NEXT: ld1 { v19.b }[5], [x11] +; CHECK-NEXT: mov v5.b[6], w6 +; CHECK-NEXT: ld1 { v17.b }[5], [x9] +; CHECK-NEXT: add x9, sp, #656 +; CHECK-NEXT: sshll v22.8h, v22.8b, #0 +; CHECK-NEXT: sshll v21.8h, v21.8b, #0 +; CHECK-NEXT: ld1 { v20.b }[5], [x9] ; CHECK-NEXT: add x10, sp, #528 -; CHECK-NEXT: add x11, sp, #464 -; CHECK-NEXT: ld1 { v20.b }[5], [x12] -; CHECK-NEXT: ld1 { v5.b }[6], [x10] -; CHECK-NEXT: add x12, sp, #728 -; CHECK-NEXT: add x13, sp, #664 -; CHECK-NEXT: add x8, sp, #72 -; CHECK-NEXT: ld1 { v17.b }[6], [x11] -; CHECK-NEXT: ld1 { v19.b }[6], [x12] -; CHECK-NEXT: ld1 { v1.b }[7], [x8] +; CHECK-NEXT: add x11, sp, #728 +; CHECK-NEXT: ld1 { v6.b }[6], [x10] +; CHECK-NEXT: add x10, sp, #464 +; CHECK-NEXT: ld1 { v19.b }[6], [x11] +; CHECK-NEXT: add x11, sp, #664 +; CHECK-NEXT: ld1 { v17.b }[6], [x10] +; CHECK-NEXT: smull v21.4s, v22.4h, v21.4h +; CHECK-NEXT: movi v22.2d, #0000000000000000 +; CHECK-NEXT: ld1 { v20.b }[6], [x11] +; CHECK-NEXT: mov v5.b[7], w7 +; CHECK-NEXT: add x9, sp, #536 +; CHECK-NEXT: add x10, sp, #736 +; CHECK-NEXT: add x11, sp, #208 +; CHECK-NEXT: add x13, sp, #264 +; CHECK-NEXT: ld1 { v6.b }[7], [x9] +; CHECK-NEXT: ld1 { v19.b }[7], [x10] +; CHECK-NEXT: ld1 { v4.b }[7], [x11] +; CHECK-NEXT: add x9, sp, #472 +; CHECK-NEXT: add x10, sp, #672 +; CHECK-NEXT: ld1 { v3.b }[6], [x13] +; CHECK-NEXT: ld1 { v17.b }[7], [x9] +; CHECK-NEXT: ld1 { v20.b }[7], [x10] ; CHECK-NEXT: add x8, sp, #336 -; CHECK-NEXT: ld1 { v20.b }[6], [x13] -; CHECK-NEXT: add x9, sp, #272 -; CHECK-NEXT: smull v18.4s, v21.4h, v18.4h +; CHECK-NEXT: mov v22.s[0], v21.s[0] ; CHECK-NEXT: movi v21.2d, #0000000000000000 -; CHECK-NEXT: add x10, sp, #536 +; CHECK-NEXT: sshll v5.8h, v5.8b, #0 +; CHECK-NEXT: sshll v6.8h, v6.8b, #0 +; CHECK-NEXT: sshll v19.8h, v19.8b, #0 ; CHECK-NEXT: ld1 { v2.b }[7], [x8] -; CHECK-NEXT: ld1 { v3.b }[7], [x9] -; CHECK-NEXT: ld1 { v5.b }[7], [x10] -; CHECK-NEXT: add x8, sp, #472 -; CHECK-NEXT: add x9, sp, #736 -; CHECK-NEXT: add x10, sp, #672 -; CHECK-NEXT: ld1 { v17.b }[7], [x8] -; CHECK-NEXT: ld1 { v19.b }[7], [x9] -; CHECK-NEXT: ld1 { v20.b }[7], [x10] +; CHECK-NEXT: add x8, sp, #272 +; CHECK-NEXT: sshll v4.8h, v4.8b, #0 +; CHECK-NEXT: ld1 { v3.b }[7], [x8] +; CHECK-NEXT: sshll v17.8h, v17.8b, #0 +; CHECK-NEXT: sshll v20.8h, v20.8b, #0 ; CHECK-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-NEXT: mov v21.s[0], v18.s[0] ; CHECK-NEXT: sshll v1.8h, v1.8b, #0 +; CHECK-NEXT: smlal v18.4s, v6.4h, v19.4h +; CHECK-NEXT: smlal2 v16.4s, v6.8h, v19.8h +; CHECK-NEXT: mov v21.s[0], v7.s[0] +; CHECK-NEXT: smull v6.4s, v5.4h, v4.4h ; CHECK-NEXT: sshll v2.8h, v2.8b, #0 ; CHECK-NEXT: sshll v3.8h, v3.8b, #0 -; CHECK-NEXT: sshll v5.8h, v5.8b, #0 -; CHECK-NEXT: sshll v17.8h, v17.8b, #0 -; CHECK-NEXT: sshll v18.8h, v19.8b, #0 -; CHECK-NEXT: sshll v19.8h, v20.8b, #0 -; CHECK-NEXT: smlal v16.4s, v0.4h, v2.4h +; CHECK-NEXT: smlal v22.4s, v17.4h, v20.4h +; CHECK-NEXT: smull2 v4.4s, v5.8h, v4.8h +; CHECK-NEXT: smlal v21.4s, v1.4h, v3.4h +; CHECK-NEXT: smlal2 v16.4s, v17.8h, v20.8h +; CHECK-NEXT: smlal v6.4s, v0.4h, v2.4h +; CHECK-NEXT: add v5.4s, v18.4s, v22.4s ; CHECK-NEXT: smlal2 v4.4s, v0.8h, v2.8h -; CHECK-NEXT: smlal v7.4s, v1.4h, v3.4h -; CHECK-NEXT: smlal v6.4s, v5.4h, v18.4h -; CHECK-NEXT: smlal2 v22.4s, v5.8h, v18.8h -; CHECK-NEXT: smlal v21.4s, v17.4h, v19.4h +; CHECK-NEXT: add v0.4s, v6.4s, v21.4s +; CHECK-NEXT: add v2.4s, v5.4s, v16.4s ; CHECK-NEXT: smlal2 v4.4s, v1.8h, v3.8h -; CHECK-NEXT: add v0.4s, v16.4s, v7.4s -; CHECK-NEXT: add v1.4s, v6.4s, v21.4s -; CHECK-NEXT: smlal2 v22.4s, v17.8h, v19.8h +; CHECK-NEXT: add v0.4s, v0.4s, v2.4s ; CHECK-NEXT: add v0.4s, v0.4s, v4.4s -; CHECK-NEXT: add v1.4s, v1.4s, v22.4s -; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: addv s0, v0.4s ; CHECK-NEXT: fmov w0, s0 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload @@ -1584,34 +1584,34 @@ entry: define i32 @test_udot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { ; CHECK-LABEL: test_udot_v33i8: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ldp q2, q3, [x0] -; CHECK-NEXT: movi v18.2d, #0000000000000000 -; CHECK-NEXT: ldp q4, q5, [x1] ; CHECK-NEXT: ldr b0, [x0, #32] ; CHECK-NEXT: ldr b1, [x1, #32] +; CHECK-NEXT: ldp q2, q4, [x0] +; CHECK-NEXT: ldp q3, q6, [x1] ; CHECK-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-NEXT: ushll v6.8h, v2.8b, #0 -; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0 ; CHECK-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-NEXT: ushll v7.8h, v4.8b, #0 -; CHECK-NEXT: ushll2 v4.8h, v4.16b, #0 -; CHECK-NEXT: ushll2 v16.8h, v3.16b, #0 -; CHECK-NEXT: ushll v3.8h, v3.8b, #0 -; CHECK-NEXT: ushll2 v19.8h, v5.16b, #0 -; CHECK-NEXT: ushll v5.8h, v5.8b, #0 +; CHECK-NEXT: ushll v5.8h, v2.8b, #0 +; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0 +; CHECK-NEXT: ushll2 v16.8h, v4.16b, #0 +; CHECK-NEXT: ushll v7.8h, v3.8b, #0 +; CHECK-NEXT: ushll2 v3.8h, v3.16b, #0 +; CHECK-NEXT: ushll v4.8h, v4.8b, #0 ; CHECK-NEXT: umull v0.4s, v1.4h, v0.4h -; CHECK-NEXT: umull2 v1.4s, v7.8h, v6.8h -; CHECK-NEXT: umull2 v17.4s, v4.8h, v2.8h -; CHECK-NEXT: umull v2.4s, v4.4h, v2.4h -; CHECK-NEXT: umlal2 v17.4s, v19.8h, v16.8h -; CHECK-NEXT: umlal2 v1.4s, v5.8h, v3.8h -; CHECK-NEXT: mov v18.s[0], v0.s[0] -; CHECK-NEXT: umlal v2.4s, v19.4h, v16.4h -; CHECK-NEXT: add v0.4s, v1.4s, v17.4s -; CHECK-NEXT: umlal v18.4s, v7.4h, v6.4h -; CHECK-NEXT: umlal v18.4s, v5.4h, v3.4h -; CHECK-NEXT: add v0.4s, v2.4s, v0.4s -; CHECK-NEXT: add v0.4s, v18.4s, v0.4s +; CHECK-NEXT: movi v1.2d, #0000000000000000 +; CHECK-NEXT: ushll2 v19.8h, v6.16b, #0 +; CHECK-NEXT: ushll v6.8h, v6.8b, #0 +; CHECK-NEXT: umull2 v17.4s, v7.8h, v5.8h +; CHECK-NEXT: umull2 v18.4s, v3.8h, v2.8h +; CHECK-NEXT: mov v1.s[0], v0.s[0] +; CHECK-NEXT: umull v0.4s, v3.4h, v2.4h +; CHECK-NEXT: umlal2 v18.4s, v19.8h, v16.8h +; CHECK-NEXT: umlal2 v17.4s, v6.8h, v4.8h +; CHECK-NEXT: umlal v1.4s, v7.4h, v5.4h +; CHECK-NEXT: umlal v0.4s, v19.4h, v16.4h +; CHECK-NEXT: add v2.4s, v17.4s, v18.4s +; CHECK-NEXT: umlal v1.4s, v6.4h, v4.4h +; CHECK-NEXT: add v0.4s, v0.4s, v2.4s +; CHECK-NEXT: add v0.4s, v1.4s, v0.4s ; CHECK-NEXT: addv s0, v0.4s ; CHECK-NEXT: fmov w8, s0 ; CHECK-NEXT: add w0, w8, w2 @@ -1639,14 +1639,14 @@ define i32 @test_udot_v33i8_nomla(ptr nocapture readonly %a1) { ; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0 ; CHECK-NEXT: ushll2 v3.8h, v3.16b, #0 ; CHECK-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-NEXT: uaddl2 v6.4s, v3.8h, v2.8h -; CHECK-NEXT: uaddl v2.4s, v3.4h, v2.4h +; CHECK-NEXT: uaddl2 v6.4s, v5.8h, v4.8h ; CHECK-NEXT: mov v0.s[0], v1.s[0] -; CHECK-NEXT: uaddl2 v1.4s, v5.8h, v4.8h -; CHECK-NEXT: add v1.4s, v1.4s, v6.4s +; CHECK-NEXT: uaddl2 v1.4s, v3.8h, v2.8h +; CHECK-NEXT: uaddl v2.4s, v3.4h, v2.4h +; CHECK-NEXT: add v1.4s, v6.4s, v1.4s ; CHECK-NEXT: uaddw v0.4s, v0.4s, v5.4h -; CHECK-NEXT: uaddw v0.4s, v0.4s, v4.4h ; CHECK-NEXT: add v1.4s, v2.4s, v1.4s +; CHECK-NEXT: uaddw v0.4s, v0.4s, v4.4h ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: addv s0, v0.4s ; CHECK-NEXT: fmov w0, s0 @@ -1660,34 +1660,34 @@ entry: define i32 @test_sdot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { ; CHECK-LABEL: test_sdot_v33i8: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ldp q2, q3, [x0] -; CHECK-NEXT: movi v18.2d, #0000000000000000 -; CHECK-NEXT: ldp q4, q5, [x1] ; CHECK-NEXT: ldr b0, [x0, #32] ; CHECK-NEXT: ldr b1, [x1, #32] +; CHECK-NEXT: ldp q2, q4, [x0] +; CHECK-NEXT: ldp q3, q6, [x1] ; CHECK-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-NEXT: sshll v6.8h, v2.8b, #0 -; CHECK-NEXT: sshll2 v2.8h, v2.16b, #0 ; CHECK-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-NEXT: sshll v7.8h, v4.8b, #0 -; CHECK-NEXT: sshll2 v4.8h, v4.16b, #0 -; CHECK-NEXT: sshll2 v16.8h, v3.16b, #0 -; CHECK-NEXT: sshll v3.8h, v3.8b, #0 -; CHECK-NEXT: sshll2 v19.8h, v5.16b, #0 -; CHECK-NEXT: sshll v5.8h, v5.8b, #0 +; CHECK-NEXT: sshll v5.8h, v2.8b, #0 +; CHECK-NEXT: sshll2 v2.8h, v2.16b, #0 +; CHECK-NEXT: sshll2 v16.8h, v4.16b, #0 +; CHECK-NEXT: sshll v7.8h, v3.8b, #0 +; CHECK-NEXT: sshll2 v3.8h, v3.16b, #0 +; CHECK-NEXT: sshll v4.8h, v4.8b, #0 ; CHECK-NEXT: smull v0.4s, v1.4h, v0.4h -; CHECK-NEXT: smull2 v1.4s, v7.8h, v6.8h -; CHECK-NEXT: smull2 v17.4s, v4.8h, v2.8h -; CHECK-NEXT: smull v2.4s, v4.4h, v2.4h -; CHECK-NEXT: smlal2 v17.4s, v19.8h, v16.8h -; CHECK-NEXT: smlal2 v1.4s, v5.8h, v3.8h -; CHECK-NEXT: mov v18.s[0], v0.s[0] -; CHECK-NEXT: smlal v2.4s, v19.4h, v16.4h -; CHECK-NEXT: add v0.4s, v1.4s, v17.4s -; CHECK-NEXT: smlal v18.4s, v7.4h, v6.4h -; CHECK-NEXT: smlal v18.4s, v5.4h, v3.4h -; CHECK-NEXT: add v0.4s, v2.4s, v0.4s -; CHECK-NEXT: add v0.4s, v18.4s, v0.4s +; CHECK-NEXT: movi v1.2d, #0000000000000000 +; CHECK-NEXT: sshll2 v19.8h, v6.16b, #0 +; CHECK-NEXT: sshll v6.8h, v6.8b, #0 +; CHECK-NEXT: smull2 v17.4s, v7.8h, v5.8h +; CHECK-NEXT: smull2 v18.4s, v3.8h, v2.8h +; CHECK-NEXT: mov v1.s[0], v0.s[0] +; CHECK-NEXT: smull v0.4s, v3.4h, v2.4h +; CHECK-NEXT: smlal2 v18.4s, v19.8h, v16.8h +; CHECK-NEXT: smlal2 v17.4s, v6.8h, v4.8h +; CHECK-NEXT: smlal v1.4s, v7.4h, v5.4h +; CHECK-NEXT: smlal v0.4s, v19.4h, v16.4h +; CHECK-NEXT: add v2.4s, v17.4s, v18.4s +; CHECK-NEXT: smlal v1.4s, v6.4h, v4.4h +; CHECK-NEXT: add v0.4s, v0.4s, v2.4s +; CHECK-NEXT: add v0.4s, v1.4s, v0.4s ; CHECK-NEXT: addv s0, v0.4s ; CHECK-NEXT: fmov w8, s0 ; CHECK-NEXT: add w0, w8, w2 @@ -2018,151 +2018,151 @@ define i32 @test_sdot_v33i8_double_nomla(<33 x i8> %a, <33 x i8> %b, <33 x i8> % ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: .cfi_def_cfa_offset 16 ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ldr b1, [sp, #80] +; CHECK-NEXT: ldr b0, [sp, #80] ; CHECK-NEXT: add x8, sp, #88 ; CHECK-NEXT: ldr b2, [sp, #144] -; CHECK-NEXT: add x9, sp, #152 -; CHECK-NEXT: ldr b3, [sp, #16] -; CHECK-NEXT: add x12, sp, #32 -; CHECK-NEXT: ld1 { v1.b }[1], [x8] -; CHECK-NEXT: ld1 { v2.b }[1], [x9] ; CHECK-NEXT: add x9, sp, #96 -; CHECK-NEXT: add x8, sp, #24 -; CHECK-NEXT: add x11, sp, #112 -; CHECK-NEXT: fmov s0, w0 -; CHECK-NEXT: ld1 { v3.b }[1], [x8] -; CHECK-NEXT: add x8, sp, #160 -; CHECK-NEXT: ldr b4, [sp, #480] -; CHECK-NEXT: ld1 { v1.b }[2], [x9] -; CHECK-NEXT: add x9, sp, #104 -; CHECK-NEXT: ld1 { v2.b }[2], [x8] +; CHECK-NEXT: add x10, sp, #152 +; CHECK-NEXT: add x11, sp, #160 +; CHECK-NEXT: ld1 { v0.b }[1], [x8] +; CHECK-NEXT: ld1 { v2.b }[1], [x10] +; CHECK-NEXT: add x8, sp, #104 +; CHECK-NEXT: ldr b3, [sp, #16] +; CHECK-NEXT: ldr b5, [sp, #480] +; CHECK-NEXT: fmov s1, w0 +; CHECK-NEXT: add x10, sp, #24 +; CHECK-NEXT: add x13, sp, #488 +; CHECK-NEXT: ldr b4, [sp, #608] +; CHECK-NEXT: ld1 { v0.b }[2], [x9] +; CHECK-NEXT: ld1 { v2.b }[2], [x11] +; CHECK-NEXT: add x12, sp, #112 +; CHECK-NEXT: ld1 { v3.b }[1], [x10] +; CHECK-NEXT: ld1 { v5.b }[1], [x13] +; CHECK-NEXT: add x10, sp, #616 +; CHECK-NEXT: mov v1.b[1], w1 +; CHECK-NEXT: ld1 { v4.b }[1], [x10] +; CHECK-NEXT: add x10, sp, #32 +; CHECK-NEXT: ld1 { v0.b }[3], [x8] ; CHECK-NEXT: add x8, sp, #168 -; CHECK-NEXT: add x10, sp, #120 -; CHECK-NEXT: add x13, sp, #48 -; CHECK-NEXT: ld1 { v3.b }[2], [x12] -; CHECK-NEXT: add x12, sp, #40 -; CHECK-NEXT: ldr b5, [sp, #608] -; CHECK-NEXT: ld1 { v1.b }[3], [x9] +; CHECK-NEXT: add x9, sp, #120 ; CHECK-NEXT: ld1 { v2.b }[3], [x8] -; CHECK-NEXT: mov v0.b[1], w1 -; CHECK-NEXT: add x9, sp, #128 -; CHECK-NEXT: add x14, sp, #184 -; CHECK-NEXT: ldr b16, [sp, #544] -; CHECK-NEXT: ld1 { v3.b }[3], [x12] -; CHECK-NEXT: add x12, sp, #176 -; CHECK-NEXT: ldr b17, [sp, #672] -; CHECK-NEXT: ld1 { v1.b }[4], [x11] -; CHECK-NEXT: add x11, sp, #488 -; CHECK-NEXT: ld1 { v2.b }[4], [x12] -; CHECK-NEXT: ld1 { v4.b }[1], [x11] -; CHECK-NEXT: mov v0.b[2], w2 -; CHECK-NEXT: add x11, sp, #192 -; CHECK-NEXT: ld1 { v3.b }[4], [x13] -; CHECK-NEXT: add x13, sp, #616 -; CHECK-NEXT: add x12, sp, #56 -; CHECK-NEXT: ld1 { v1.b }[5], [x10] -; CHECK-NEXT: ld1 { v5.b }[1], [x13] -; CHECK-NEXT: add x13, sp, #496 +; CHECK-NEXT: add x8, sp, #496 +; CHECK-NEXT: ld1 { v3.b }[2], [x10] +; CHECK-NEXT: ld1 { v5.b }[2], [x8] +; CHECK-NEXT: add x8, sp, #176 +; CHECK-NEXT: ldr b6, [sp, #544] +; CHECK-NEXT: ld1 { v0.b }[4], [x12] +; CHECK-NEXT: add x14, sp, #552 +; CHECK-NEXT: ldr b7, [sp, #672] +; CHECK-NEXT: ld1 { v2.b }[4], [x8] +; CHECK-NEXT: add x13, sp, #40 +; CHECK-NEXT: ld1 { v6.b }[1], [x14] +; CHECK-NEXT: mov v1.b[2], w2 +; CHECK-NEXT: add x11, sp, #128 +; CHECK-NEXT: ld1 { v3.b }[3], [x13] +; CHECK-NEXT: ld1 { v0.b }[5], [x9] +; CHECK-NEXT: add x9, sp, #680 +; CHECK-NEXT: add x13, sp, #184 +; CHECK-NEXT: ld1 { v7.b }[1], [x9] +; CHECK-NEXT: ld1 { v2.b }[5], [x13] +; CHECK-NEXT: add x13, sp, #624 +; CHECK-NEXT: add x15, sp, #504 ; CHECK-NEXT: ld1 { v4.b }[2], [x13] -; CHECK-NEXT: ld1 { v2.b }[5], [x14] -; CHECK-NEXT: add x14, sp, #680 -; CHECK-NEXT: ld1 { v17.b }[1], [x14] -; CHECK-NEXT: add x13, sp, #504 -; CHECK-NEXT: ld1 { v3.b }[5], [x12] -; CHECK-NEXT: ld1 { v1.b }[6], [x9] -; CHECK-NEXT: add x9, sp, #552 -; CHECK-NEXT: add x12, sp, #688 -; CHECK-NEXT: ld1 { v16.b }[1], [x9] -; CHECK-NEXT: add x9, sp, #624 -; CHECK-NEXT: ld1 { v4.b }[3], [x13] -; CHECK-NEXT: ld1 { v2.b }[6], [x11] +; CHECK-NEXT: add x10, sp, #136 +; CHECK-NEXT: ld1 { v0.b }[6], [x11] ; CHECK-NEXT: add x11, sp, #560 -; CHECK-NEXT: add x8, sp, #136 -; CHECK-NEXT: ld1 { v17.b }[2], [x12] -; CHECK-NEXT: ld1 { v5.b }[2], [x9] -; CHECK-NEXT: ld1 { v1.b }[7], [x8] -; CHECK-NEXT: ld1 { v16.b }[2], [x11] -; CHECK-NEXT: add x8, sp, #512 -; CHECK-NEXT: mov v0.b[3], w3 -; CHECK-NEXT: ld1 { v4.b }[4], [x8] -; CHECK-NEXT: add x8, sp, #568 -; CHECK-NEXT: add x9, sp, #696 -; CHECK-NEXT: add x11, sp, #632 -; CHECK-NEXT: ld1 { v17.b }[3], [x9] -; CHECK-NEXT: add x9, sp, #520 -; CHECK-NEXT: ld1 { v16.b }[3], [x8] -; CHECK-NEXT: ld1 { v5.b }[3], [x11] -; CHECK-NEXT: add x8, sp, #640 -; CHECK-NEXT: ld1 { v4.b }[5], [x9] +; CHECK-NEXT: ld1 { v5.b }[3], [x15] +; CHECK-NEXT: ld1 { v6.b }[2], [x11] +; CHECK-NEXT: add x11, sp, #688 +; CHECK-NEXT: mov v1.b[3], w3 +; CHECK-NEXT: ld1 { v7.b }[2], [x11] +; CHECK-NEXT: add x9, sp, #632 +; CHECK-NEXT: add x11, sp, #512 +; CHECK-NEXT: ld1 { v0.b }[7], [x10] +; CHECK-NEXT: ld1 { v4.b }[3], [x9] +; CHECK-NEXT: add x9, sp, #568 +; CHECK-NEXT: add x10, sp, #696 +; CHECK-NEXT: ld1 { v6.b }[3], [x9] +; CHECK-NEXT: ld1 { v5.b }[4], [x11] +; CHECK-NEXT: ld1 { v7.b }[3], [x10] +; CHECK-NEXT: add x9, sp, #640 +; CHECK-NEXT: mov v1.b[4], w4 +; CHECK-NEXT: ld1 { v4.b }[4], [x9] ; CHECK-NEXT: add x9, sp, #576 -; CHECK-NEXT: add x11, sp, #704 +; CHECK-NEXT: add x10, sp, #704 +; CHECK-NEXT: add x11, sp, #520 +; CHECK-NEXT: ld1 { v6.b }[4], [x9] ; CHECK-NEXT: ldr b18, [sp, #736] -; CHECK-NEXT: mov v0.b[4], w4 -; CHECK-NEXT: ld1 { v17.b }[4], [x11] -; CHECK-NEXT: ld1 { v16.b }[4], [x9] -; CHECK-NEXT: ld1 { v5.b }[4], [x8] -; CHECK-NEXT: add x9, sp, #528 -; CHECK-NEXT: sshll v18.8h, v18.8b, #0 -; CHECK-NEXT: add x8, sp, #648 +; CHECK-NEXT: ld1 { v7.b }[4], [x10] +; CHECK-NEXT: ld1 { v5.b }[5], [x11] +; CHECK-NEXT: add x12, sp, #192 +; CHECK-NEXT: add x8, sp, #48 +; CHECK-NEXT: ld1 { v2.b }[6], [x12] +; CHECK-NEXT: add x9, sp, #648 +; CHECK-NEXT: ld1 { v3.b }[4], [x8] +; CHECK-NEXT: add x10, sp, #528 ; CHECK-NEXT: add x11, sp, #584 ; CHECK-NEXT: add x12, sp, #712 -; CHECK-NEXT: ld1 { v4.b }[6], [x9] -; CHECK-NEXT: movi v7.2d, #0000000000000000 -; CHECK-NEXT: ld1 { v16.b }[5], [x11] -; CHECK-NEXT: ld1 { v17.b }[5], [x12] -; CHECK-NEXT: ld1 { v5.b }[5], [x8] -; CHECK-NEXT: mov v0.b[5], w5 -; CHECK-NEXT: add x9, sp, #536 -; CHECK-NEXT: sshll v18.4s, v18.4h, #0 -; CHECK-NEXT: add x8, sp, #656 +; CHECK-NEXT: sshll v18.8h, v18.8b, #0 +; CHECK-NEXT: mov v1.b[5], w5 +; CHECK-NEXT: ld1 { v6.b }[5], [x11] +; CHECK-NEXT: ld1 { v7.b }[5], [x12] +; CHECK-NEXT: ld1 { v4.b }[5], [x9] +; CHECK-NEXT: ld1 { v5.b }[6], [x10] +; CHECK-NEXT: add x14, sp, #56 +; CHECK-NEXT: movi v17.2d, #0000000000000000 +; CHECK-NEXT: ld1 { v3.b }[5], [x14] +; CHECK-NEXT: add x9, sp, #656 +; CHECK-NEXT: add x10, sp, #536 ; CHECK-NEXT: add x11, sp, #592 ; CHECK-NEXT: add x12, sp, #720 -; CHECK-NEXT: ld1 { v4.b }[7], [x9] -; CHECK-NEXT: ld1 { v16.b }[6], [x11] -; CHECK-NEXT: ld1 { v17.b }[6], [x12] -; CHECK-NEXT: ld1 { v5.b }[6], [x8] -; CHECK-NEXT: ldr b6, [sp, #208] -; CHECK-NEXT: add x10, sp, #64 -; CHECK-NEXT: mov v7.s[0], v18.s[0] -; CHECK-NEXT: mov v0.b[6], w6 -; CHECK-NEXT: ld1 { v3.b }[6], [x10] +; CHECK-NEXT: sshll v18.4s, v18.4h, #0 +; CHECK-NEXT: ldr b16, [sp, #208] +; CHECK-NEXT: ld1 { v6.b }[6], [x11] +; CHECK-NEXT: ld1 { v7.b }[6], [x12] +; CHECK-NEXT: ld1 { v4.b }[6], [x9] +; CHECK-NEXT: ld1 { v5.b }[7], [x10] +; CHECK-NEXT: add x8, sp, #64 +; CHECK-NEXT: mov v1.b[6], w6 +; CHECK-NEXT: sshll v16.8h, v16.8b, #0 +; CHECK-NEXT: ld1 { v3.b }[6], [x8] ; CHECK-NEXT: add x8, sp, #664 ; CHECK-NEXT: add x9, sp, #600 ; CHECK-NEXT: add x10, sp, #728 -; CHECK-NEXT: sshll v4.8h, v4.8b, #0 -; CHECK-NEXT: sshll v6.8h, v6.8b, #0 -; CHECK-NEXT: ld1 { v16.b }[7], [x9] -; CHECK-NEXT: ld1 { v17.b }[7], [x10] -; CHECK-NEXT: ld1 { v5.b }[7], [x8] +; CHECK-NEXT: mov v17.s[0], v18.s[0] +; CHECK-NEXT: ld1 { v6.b }[7], [x9] +; CHECK-NEXT: ld1 { v7.b }[7], [x10] +; CHECK-NEXT: ld1 { v4.b }[7], [x8] +; CHECK-NEXT: sshll v5.8h, v5.8b, #0 ; CHECK-NEXT: movi v18.2d, #0000000000000000 -; CHECK-NEXT: mov v0.b[7], w7 +; CHECK-NEXT: sshll v16.4s, v16.4h, #0 +; CHECK-NEXT: mov v1.b[7], w7 ; CHECK-NEXT: add x9, sp, #200 -; CHECK-NEXT: add x10, sp, #72 -; CHECK-NEXT: saddw v7.4s, v7.4s, v4.4h -; CHECK-NEXT: sshll v6.4s, v6.4h, #0 -; CHECK-NEXT: sshll v16.8h, v16.8b, #0 -; CHECK-NEXT: sshll v17.8h, v17.8b, #0 -; CHECK-NEXT: sshll v5.8h, v5.8b, #0 +; CHECK-NEXT: add x8, sp, #72 ; CHECK-NEXT: ld1 { v2.b }[7], [x9] -; CHECK-NEXT: ld1 { v3.b }[7], [x10] -; CHECK-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-NEXT: mov v18.s[0], v6.s[0] +; CHECK-NEXT: sshll v6.8h, v6.8b, #0 +; CHECK-NEXT: ld1 { v3.b }[7], [x8] +; CHECK-NEXT: sshll v7.8h, v7.8b, #0 +; CHECK-NEXT: sshll v4.8h, v4.8b, #0 +; CHECK-NEXT: saddw v17.4s, v17.4s, v5.4h +; CHECK-NEXT: mov v18.s[0], v16.s[0] ; CHECK-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-NEXT: saddl2 v6.4s, v17.8h, v16.8h -; CHECK-NEXT: saddl2 v4.4s, v5.8h, v4.8h -; CHECK-NEXT: saddl v16.4s, v17.4h, v16.4h -; CHECK-NEXT: saddw v5.4s, v7.4s, v5.4h +; CHECK-NEXT: sshll v1.8h, v1.8b, #0 ; CHECK-NEXT: sshll v2.8h, v2.8b, #0 ; CHECK-NEXT: sshll v3.8h, v3.8b, #0 -; CHECK-NEXT: saddl2 v17.4s, v0.8h, v1.8h -; CHECK-NEXT: saddw v0.4s, v18.4s, v0.4h +; CHECK-NEXT: saddl2 v16.4s, v7.8h, v6.8h +; CHECK-NEXT: saddl2 v5.4s, v4.8h, v5.8h +; CHECK-NEXT: saddl v6.4s, v7.4h, v6.4h +; CHECK-NEXT: saddw v4.4s, v17.4s, v4.4h +; CHECK-NEXT: saddl2 v17.4s, v1.8h, v0.8h +; CHECK-NEXT: saddw v1.4s, v18.4s, v1.4h ; CHECK-NEXT: saddl2 v7.4s, v3.8h, v2.8h -; CHECK-NEXT: add v4.4s, v4.4s, v6.4s -; CHECK-NEXT: saddl v2.4s, v3.4h, v2.4h ; CHECK-NEXT: add v5.4s, v5.4s, v16.4s -; CHECK-NEXT: saddw v0.4s, v0.4s, v1.4h +; CHECK-NEXT: saddl v2.4s, v3.4h, v2.4h +; CHECK-NEXT: add v4.4s, v4.4s, v6.4s +; CHECK-NEXT: saddw v0.4s, v1.4s, v0.4h ; CHECK-NEXT: add v6.4s, v17.4s, v7.4s -; CHECK-NEXT: add v1.4s, v5.4s, v4.4s +; CHECK-NEXT: add v1.4s, v4.4s, v5.4s ; CHECK-NEXT: add v0.4s, v0.4s, v2.4s ; CHECK-NEXT: add v1.4s, v6.4s, v1.4s ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s diff --git a/llvm/test/CodeGen/AArch64/neon-extadd.ll b/llvm/test/CodeGen/AArch64/neon-extadd.ll index 913205f32753..16200435c5c3 100644 --- a/llvm/test/CodeGen/AArch64/neon-extadd.ll +++ b/llvm/test/CodeGen/AArch64/neon-extadd.ll @@ -515,15 +515,15 @@ define <20 x i32> @v20(<20 x i8> %s0, <20 x i8> %s1) { ; CHECK-NEXT: mov v0.b[6], w6 ; CHECK-NEXT: ld1 { v1.b }[7], [x9] ; CHECK-NEXT: uaddl v2.8h, v3.8b, v2.8b +; CHECK-NEXT: ushll v3.4s, v4.4h, #0 ; CHECK-NEXT: mov v0.b[7], w7 -; CHECK-NEXT: ushll2 v3.4s, v2.8h, #0 -; CHECK-NEXT: ushll v2.4s, v2.4h, #0 ; CHECK-NEXT: uaddl v0.8h, v0.8b, v1.8b -; CHECK-NEXT: ushll v1.4s, v4.4h, #0 -; CHECK-NEXT: stp q3, q1, [x8, #48] -; CHECK-NEXT: ushll2 v1.4s, v0.8h, #0 +; CHECK-NEXT: ushll2 v1.4s, v2.8h, #0 +; CHECK-NEXT: ushll v2.4s, v2.4h, #0 +; CHECK-NEXT: stp q1, q3, [x8, #48] +; CHECK-NEXT: ushll2 v3.4s, v0.8h, #0 ; CHECK-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-NEXT: stp q1, q2, [x8, #16] +; CHECK-NEXT: stp q3, q2, [x8, #16] ; CHECK-NEXT: str q0, [x8] ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/AArch64/neon-shift-neg.ll b/llvm/test/CodeGen/AArch64/neon-shift-neg.ll index 881bbf315e8e..45272143e859 100644 --- a/llvm/test/CodeGen/AArch64/neon-shift-neg.ll +++ b/llvm/test/CodeGen/AArch64/neon-shift-neg.ll @@ -375,8 +375,8 @@ entry: define @shrn64x2( %a, i64 %b) { ; CHECK-LABEL: shrn64x2: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: neg x8, x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: asr z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret @@ -391,8 +391,8 @@ entry: define @shrn32x4( %a, i32 %b) { ; CHECK-LABEL: shrn32x4: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: neg w8, w0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: asr z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret @@ -407,8 +407,8 @@ entry: define @shrn16x8( %a, i16 %b) { ; CHECK-LABEL: shrn16x8: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: neg w8, w0 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: asr z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret @@ -423,8 +423,8 @@ entry: define @shrn8x16( %a, i8 %b) { ; CHECK-LABEL: shrn8x16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: neg w8, w0 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z1.b, w8 ; CHECK-NEXT: asr z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: ret @@ -439,8 +439,8 @@ entry: define @lshrn64x2( %a, i64 %b) { ; CHECK-LABEL: lshrn64x2: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: neg x8, x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: lsr z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret @@ -455,8 +455,8 @@ entry: define @lshrn32x4( %a, i32 %b) { ; CHECK-LABEL: lshrn32x4: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: neg w8, w0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: lsr z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret @@ -471,8 +471,8 @@ entry: define @lshrn16x8( %a, i16 %b) { ; CHECK-LABEL: lshrn16x8: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: neg w8, w0 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: lsr z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret @@ -487,8 +487,8 @@ entry: define @lshrn8x16( %a, i8 %b) { ; CHECK-LABEL: lshrn8x16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: neg w8, w0 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z1.b, w8 ; CHECK-NEXT: lsr z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: ret @@ -503,8 +503,8 @@ entry: define @shln64x2( %a, i64 %b) { ; CHECK-LABEL: shln64x2: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: neg x8, x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: lsl z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret @@ -519,8 +519,8 @@ entry: define @shln32x4( %a, i32 %b) { ; CHECK-LABEL: shln32x4: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: neg w8, w0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: lsl z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret @@ -535,8 +535,8 @@ entry: define @shln16x8( %a, i16 %b) { ; CHECK-LABEL: shln16x8: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: neg w8, w0 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: lsl z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret @@ -551,8 +551,8 @@ entry: define @shln8x16( %a, i8 %b) { ; CHECK-LABEL: shln8x16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: neg w8, w0 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z1.b, w8 ; CHECK-NEXT: lsl z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/predicated-add-sub.ll b/llvm/test/CodeGen/AArch64/predicated-add-sub.ll index 6b3cfc040cb3..20088354bdb7 100644 --- a/llvm/test/CodeGen/AArch64/predicated-add-sub.ll +++ b/llvm/test/CodeGen/AArch64/predicated-add-sub.ll @@ -83,11 +83,11 @@ define @zext.add.2xi64( %a, @zext.add.8xi32( %a, %v) #0 { ; CHECK-LABEL: zext.add.8xi32: ; CHECK: // %bb.0: -; CHECK-NEXT: punpkhi p1.h, p0.b ; CHECK-NEXT: mov z2.s, #1 // =0x1 +; CHECK-NEXT: punpkhi p1.h, p0.b ; CHECK-NEXT: punpklo p0.h, p0.b -; CHECK-NEXT: add z1.s, p1/m, z1.s, z2.s ; CHECK-NEXT: add z0.s, p0/m, z0.s, z2.s +; CHECK-NEXT: add z1.s, p1/m, z1.s, z2.s ; CHECK-NEXT: ret %extend = zext %v to %result = add %a, %extend @@ -103,8 +103,8 @@ define @zext.add.16xi32( %a, @zext.sub.2xi64( %a, @zext.sub.8xi32( %a, %v) #0 { ; CHECK-LABEL: zext.sub.8xi32: ; CHECK: // %bb.0: -; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: mov z2.s, #-1 // =0xffffffffffffffff +; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: add z0.s, p1/m, z0.s, z2.s ; CHECK-NEXT: add z1.s, p0/m, z1.s, z2.s @@ -214,8 +214,8 @@ define @zext.sub.16xi32( %a, @sext.add.2xi64( %a, @sext.add.8xi32( %a, %v) #0 { ; CHECK-LABEL: sext.add.8xi32: ; CHECK: // %bb.0: -; CHECK-NEXT: punpkhi p1.h, p0.b ; CHECK-NEXT: mov z2.s, #-1 // =0xffffffffffffffff +; CHECK-NEXT: punpkhi p1.h, p0.b ; CHECK-NEXT: punpklo p0.h, p0.b -; CHECK-NEXT: add z1.s, p1/m, z1.s, z2.s ; CHECK-NEXT: add z0.s, p0/m, z0.s, z2.s +; CHECK-NEXT: add z1.s, p1/m, z1.s, z2.s ; CHECK-NEXT: ret %extend = sext %v to %result = add %a, %extend @@ -325,8 +325,8 @@ define @sext.add.16xi32( %a, @sext.sub.2xi64( %a, @sext.sub.8xi32( %a, %v) #0 { ; CHECK-LABEL: sext.sub.8xi32: ; CHECK: // %bb.0: -; CHECK-NEXT: punpkhi p1.h, p0.b ; CHECK-NEXT: mov z2.s, #-1 // =0xffffffffffffffff +; CHECK-NEXT: punpkhi p1.h, p0.b ; CHECK-NEXT: punpklo p0.h, p0.b -; CHECK-NEXT: sub z1.s, p1/m, z1.s, z2.s ; CHECK-NEXT: sub z0.s, p0/m, z0.s, z2.s +; CHECK-NEXT: sub z1.s, p1/m, z1.s, z2.s ; CHECK-NEXT: ret %extend = sext %v to %result = sub %a, %extend @@ -436,8 +436,8 @@ define @sext.sub.16xi32( %a, This Inner Loop Header: Depth=1 +; CHECK-NEXT: str q14, [sp, #32] // 16-byte Folded Spill ; CHECK-NEXT: ldr q14, [x8] ; CHECK-NEXT: mov x12, xzr -; CHECK-NEXT: str q18, [sp, #32] // 16-byte Folded Spill ; CHECK-NEXT: ldr x14, [x12] -; CHECK-NEXT: ldr q15, [x12] -; CHECK-NEXT: add x7, x11, x8 +; CHECK-NEXT: stp q29, q15, [sp] // 32-byte Folded Spill +; CHECK-NEXT: add x19, x11, x8 ; CHECK-NEXT: fmov x15, d14 ; CHECK-NEXT: mov x16, v14.d[1] -; CHECK-NEXT: ldr q18, [sp, #64] // 16-byte Folded Reload -; CHECK-NEXT: fmov x18, d15 -; CHECK-NEXT: mov x13, v15.d[1] -; CHECK-NEXT: ldr x5, [x8] +; CHECK-NEXT: ldr q15, [x12] ; CHECK-NEXT: ldr q14, [x10], #64 -; CHECK-NEXT: ldr x7, [x7, #128] +; CHECK-NEXT: mov v8.16b, v28.16b +; CHECK-NEXT: fmov x13, d15 +; CHECK-NEXT: mov x18, v15.d[1] +; CHECK-NEXT: mov v28.16b, v24.16b ; CHECK-NEXT: mul x17, x15, x14 -; CHECK-NEXT: mov v6.16b, v0.16b -; CHECK-NEXT: mov v9.16b, v27.16b ; CHECK-NEXT: mov x12, v14.d[1] ; CHECK-NEXT: fmov x4, d14 -; CHECK-NEXT: mov v27.16b, v23.16b +; CHECK-NEXT: mov v24.16b, v20.16b +; CHECK-NEXT: mov v20.16b, v17.16b +; CHECK-NEXT: mov v17.16b, v5.16b ; CHECK-NEXT: mul x1, x16, x14 -; CHECK-NEXT: mov v23.16b, v19.16b -; CHECK-NEXT: mov v19.16b, v7.16b -; CHECK-NEXT: mov v7.16b, v2.16b -; CHECK-NEXT: stp q26, q31, [sp] // 32-byte Folded Spill -; CHECK-NEXT: mov v31.16b, v22.16b -; CHECK-NEXT: mul x0, x18, x14 -; CHECK-NEXT: mov v26.16b, v10.16b -; CHECK-NEXT: mov v22.16b, v5.16b +; CHECK-NEXT: ldr q5, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: ldr x5, [x8] +; CHECK-NEXT: ldr x19, [x19, #128] +; CHECK-NEXT: mov v29.16b, v21.16b +; CHECK-NEXT: mov v21.16b, v0.16b +; CHECK-NEXT: mul x0, x13, x14 +; CHECK-NEXT: mov v25.16b, v6.16b +; CHECK-NEXT: mov v6.16b, v2.16b ; CHECK-NEXT: fmov d15, x17 -; CHECK-NEXT: mov v5.16b, v1.16b -; CHECK-NEXT: mov v8.16b, v20.16b -; CHECK-NEXT: mul x2, x13, x14 -; CHECK-NEXT: mov v20.16b, v16.16b -; CHECK-NEXT: mov v16.16b, v3.16b -; CHECK-NEXT: mov v10.16b, v21.16b -; CHECK-NEXT: mov v21.16b, v17.16b -; CHECK-NEXT: mov v17.16b, v4.16b +; CHECK-NEXT: mov v26.16b, v22.16b +; CHECK-NEXT: mov v22.16b, v18.16b +; CHECK-NEXT: mul x2, x18, x14 +; CHECK-NEXT: mov v18.16b, v7.16b +; CHECK-NEXT: mov v7.16b, v3.16b +; CHECK-NEXT: mov v16.16b, v4.16b +; CHECK-NEXT: add x8, x8, #8 +; CHECK-NEXT: add x9, x9, #1 ; CHECK-NEXT: mov v15.d[1], x1 ; CHECK-NEXT: mul x3, x12, x14 -; CHECK-NEXT: add x8, x8, #8 -; CHECK-NEXT: fmov d14, x0 ; CHECK-NEXT: cmp x8, #64 -; CHECK-NEXT: add x9, x9, #1 +; CHECK-NEXT: fmov d14, x0 ; CHECK-NEXT: mul x14, x4, x14 -; CHECK-NEXT: add v18.2d, v18.2d, v15.2d -; CHECK-NEXT: mul x19, x15, x5 +; CHECK-NEXT: add v5.2d, v5.2d, v15.2d +; CHECK-NEXT: mul x20, x15, x5 ; CHECK-NEXT: mov v14.d[1], x2 -; CHECK-NEXT: mul x15, x15, x7 +; CHECK-NEXT: mul x15, x15, x19 ; CHECK-NEXT: fmov d0, x14 -; CHECK-NEXT: str q18, [sp, #64] // 16-byte Folded Spill -; CHECK-NEXT: ldp q18, q15, [sp, #32] // 32-byte Folded Reload -; CHECK-NEXT: mul x6, x16, x5 -; CHECK-NEXT: fmov d1, x19 +; CHECK-NEXT: str q5, [sp, #64] // 16-byte Folded Spill +; CHECK-NEXT: ldr q5, [sp, #48] // 16-byte Folded Reload +; CHECK-NEXT: mul x21, x13, x19 +; CHECK-NEXT: add v5.2d, v5.2d, v14.2d +; CHECK-NEXT: fmov d3, x20 +; CHECK-NEXT: mul x7, x16, x5 ; CHECK-NEXT: mov v0.d[1], x3 -; CHECK-NEXT: mul x16, x16, x7 -; CHECK-NEXT: fmov d2, x15 -; CHECK-NEXT: add v15.2d, v15.2d, v14.2d -; CHECK-NEXT: mul x21, x18, x7 -; CHECK-NEXT: mov v1.d[1], x6 -; CHECK-NEXT: mul x0, x4, x7 -; CHECK-NEXT: str q15, [sp, #48] // 16-byte Folded Spill -; CHECK-NEXT: add v15.2d, v11.2d, v14.2d -; CHECK-NEXT: mov v2.d[1], x16 -; CHECK-NEXT: ldr q11, [sp, #80] // 16-byte Folded Reload -; CHECK-NEXT: mul x20, x13, x7 -; CHECK-NEXT: fmov d3, x21 -; CHECK-NEXT: add v11.2d, v11.2d, v0.2d -; CHECK-NEXT: add v12.2d, v12.2d, v1.2d -; CHECK-NEXT: mul x22, x12, x7 -; CHECK-NEXT: fmov d4, x0 -; CHECK-NEXT: add v18.2d, v18.2d, v2.2d -; CHECK-NEXT: mov v2.16b, v7.16b -; CHECK-NEXT: mul x14, x18, x5 -; CHECK-NEXT: mov v7.16b, v19.16b -; CHECK-NEXT: mov v19.16b, v23.16b -; CHECK-NEXT: mov v3.d[1], x20 -; CHECK-NEXT: mov v23.16b, v27.16b -; CHECK-NEXT: add v27.2d, v9.2d, v1.2d -; CHECK-NEXT: mul x15, x4, x5 -; CHECK-NEXT: str q11, [sp, #80] // 16-byte Folded Spill -; CHECK-NEXT: mov v11.16b, v15.16b -; CHECK-NEXT: mov v4.d[1], x22 -; CHECK-NEXT: add v19.2d, v19.2d, v1.2d -; CHECK-NEXT: add v7.2d, v7.2d, v1.2d +; CHECK-NEXT: fmov d1, x15 +; CHECK-NEXT: mul x16, x16, x19 +; CHECK-NEXT: str q5, [sp, #48] // 16-byte Folded Spill +; CHECK-NEXT: add v5.2d, v13.2d, v14.2d +; CHECK-NEXT: fmov d2, x21 +; CHECK-NEXT: ldr q13, [sp, #80] // 16-byte Folded Reload +; CHECK-NEXT: mul x6, x18, x5 +; CHECK-NEXT: ldp q15, q14, [sp, #16] // 32-byte Folded Reload +; CHECK-NEXT: mov v3.d[1], x7 +; CHECK-NEXT: add v13.2d, v13.2d, v0.2d +; CHECK-NEXT: mul x18, x18, x19 +; CHECK-NEXT: mov v1.d[1], x16 +; CHECK-NEXT: mul x22, x4, x19 +; CHECK-NEXT: str q13, [sp, #80] // 16-byte Folded Spill +; CHECK-NEXT: mov v13.16b, v5.16b +; CHECK-NEXT: mov v5.16b, v17.16b +; CHECK-NEXT: mov v17.16b, v20.16b +; CHECK-NEXT: mov v20.16b, v24.16b ; CHECK-NEXT: mul x13, x13, x5 -; CHECK-NEXT: add v23.2d, v23.2d, v1.2d -; CHECK-NEXT: add v1.2d, v5.2d, v1.2d -; CHECK-NEXT: fmov d14, x14 -; CHECK-NEXT: add v30.2d, v30.2d, v3.2d -; CHECK-NEXT: mov v3.16b, v16.16b +; CHECK-NEXT: mov v24.16b, v28.16b +; CHECK-NEXT: add v11.2d, v11.2d, v3.2d +; CHECK-NEXT: mov v2.d[1], x18 +; CHECK-NEXT: add v15.2d, v15.2d, v1.2d +; CHECK-NEXT: add v27.2d, v27.2d, v3.2d +; CHECK-NEXT: mul x17, x12, x19 +; CHECK-NEXT: add v23.2d, v23.2d, v3.2d +; CHECK-NEXT: add v19.2d, v19.2d, v3.2d +; CHECK-NEXT: fmov d4, x22 +; CHECK-NEXT: add v10.2d, v10.2d, v3.2d +; CHECK-NEXT: mul x14, x4, x5 +; CHECK-NEXT: fmov d0, x13 +; CHECK-NEXT: add v14.2d, v14.2d, v2.2d +; CHECK-NEXT: add v2.2d, v6.2d, v3.2d ; CHECK-NEXT: mul x12, x12, x5 -; CHECK-NEXT: mov v16.16b, v20.16b -; CHECK-NEXT: mov v5.16b, v22.16b -; CHECK-NEXT: fmov d0, x15 -; CHECK-NEXT: add v28.2d, v28.2d, v4.2d -; CHECK-NEXT: mov v4.16b, v17.16b -; CHECK-NEXT: mov v17.16b, v21.16b -; CHECK-NEXT: mov v21.16b, v10.16b -; CHECK-NEXT: mov v10.16b, v26.16b -; CHECK-NEXT: mov v14.d[1], x13 -; CHECK-NEXT: mov v22.16b, v31.16b -; CHECK-NEXT: ldp q26, q31, [sp] // 32-byte Folded Reload -; CHECK-NEXT: mov v0.d[1], x12 -; CHECK-NEXT: add v13.2d, v13.2d, v14.2d -; CHECK-NEXT: add v31.2d, v31.2d, v14.2d -; CHECK-NEXT: add v26.2d, v26.2d, v14.2d -; CHECK-NEXT: add v24.2d, v24.2d, v14.2d -; CHECK-NEXT: add v22.2d, v22.2d, v14.2d -; CHECK-NEXT: add v20.2d, v8.2d, v14.2d -; CHECK-NEXT: add v10.2d, v10.2d, v14.2d -; CHECK-NEXT: add v16.2d, v16.2d, v14.2d -; CHECK-NEXT: add v5.2d, v5.2d, v14.2d -; CHECK-NEXT: add v3.2d, v3.2d, v14.2d -; CHECK-NEXT: add v2.2d, v2.2d, v14.2d -; CHECK-NEXT: add v29.2d, v29.2d, v0.2d -; CHECK-NEXT: add v25.2d, v25.2d, v0.2d -; CHECK-NEXT: add v21.2d, v21.2d, v0.2d +; CHECK-NEXT: mov v3.16b, v7.16b +; CHECK-NEXT: mov v7.16b, v18.16b +; CHECK-NEXT: mov v4.d[1], x17 +; CHECK-NEXT: mov v18.16b, v22.16b +; CHECK-NEXT: mov v0.d[1], x6 +; CHECK-NEXT: fmov d1, x14 +; CHECK-NEXT: add v28.2d, v8.2d, v4.2d +; CHECK-NEXT: mov v1.d[1], x12 +; CHECK-NEXT: add v31.2d, v31.2d, v0.2d +; CHECK-NEXT: add v30.2d, v30.2d, v0.2d +; CHECK-NEXT: add v12.2d, v12.2d, v0.2d +; CHECK-NEXT: add v24.2d, v24.2d, v0.2d +; CHECK-NEXT: add v22.2d, v26.2d, v0.2d +; CHECK-NEXT: add v20.2d, v20.2d, v0.2d +; CHECK-NEXT: add v18.2d, v18.2d, v0.2d ; CHECK-NEXT: add v17.2d, v17.2d, v0.2d -; CHECK-NEXT: add v4.2d, v4.2d, v0.2d -; CHECK-NEXT: add v0.2d, v6.2d, v0.2d +; CHECK-NEXT: add v7.2d, v7.2d, v0.2d +; CHECK-NEXT: add v4.2d, v16.2d, v0.2d +; CHECK-NEXT: add v3.2d, v3.2d, v0.2d +; CHECK-NEXT: mov v0.16b, v21.16b +; CHECK-NEXT: mov v21.16b, v29.16b +; CHECK-NEXT: ldr q29, [sp] // 16-byte Folded Reload +; CHECK-NEXT: add v9.2d, v9.2d, v1.2d +; CHECK-NEXT: add v6.2d, v25.2d, v1.2d +; CHECK-NEXT: add v5.2d, v5.2d, v1.2d +; CHECK-NEXT: add v29.2d, v29.2d, v1.2d +; CHECK-NEXT: add v21.2d, v21.2d, v1.2d +; CHECK-NEXT: add v0.2d, v0.2d, v1.2d ; CHECK-NEXT: b.ne .LBB0_1 ; CHECK-NEXT: // %bb.2: // %for.cond.cleanup -; CHECK-NEXT: ldr q6, [sp, #48] // 16-byte Folded Reload +; CHECK-NEXT: ldr q1, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: adrp x8, C ; CHECK-NEXT: add x8, x8, :lo12:C -; CHECK-NEXT: stp q12, q31, [x8, #80] +; CHECK-NEXT: stp q11, q30, [x8, #80] ; CHECK-NEXT: ldp x20, x19, [sp, #176] // 16-byte Folded Reload -; CHECK-NEXT: str q6, [x8] -; CHECK-NEXT: ldr q6, [sp, #64] // 16-byte Folded Reload -; CHECK-NEXT: str q29, [x8, #112] +; CHECK-NEXT: str q1, [x8] +; CHECK-NEXT: ldr q1, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: stp q15, q14, [x8, #144] ; CHECK-NEXT: ldp x22, x21, [sp, #160] // 16-byte Folded Reload -; CHECK-NEXT: stp q6, q11, [x8, #16] -; CHECK-NEXT: ldr q6, [sp, #80] // 16-byte Folded Reload -; CHECK-NEXT: stp q18, q30, [x8, #144] -; CHECK-NEXT: ldp d9, d8, [sp, #144] // 16-byte Folded Reload -; CHECK-NEXT: stp q6, q13, [x8, #48] +; CHECK-NEXT: stp q1, q13, [x8, #16] +; CHECK-NEXT: ldr q1, [sp, #80] // 16-byte Folded Reload +; CHECK-NEXT: stp q28, q12, [x8, #176] ; CHECK-NEXT: ldp d13, d12, [sp, #112] // 16-byte Folded Reload -; CHECK-NEXT: stp q28, q26, [x8, #176] +; CHECK-NEXT: stp q1, q31, [x8, #48] ; CHECK-NEXT: ldp d15, d14, [sp, #96] // 16-byte Folded Reload -; CHECK-NEXT: stp q19, q10, [x8, #336] +; CHECK-NEXT: stp q9, q24, [x8, #240] +; CHECK-NEXT: ldp d9, d8, [sp, #144] // 16-byte Folded Reload +; CHECK-NEXT: stp q19, q18, [x8, #336] +; CHECK-NEXT: stp q10, q7, [x8, #400] ; CHECK-NEXT: ldp d11, d10, [sp, #128] // 16-byte Folded Reload +; CHECK-NEXT: str q29, [x8, #112] ; CHECK-NEXT: str q27, [x8, #208] -; CHECK-NEXT: stp q25, q24, [x8, #240] ; CHECK-NEXT: stp q23, q22, [x8, #272] ; CHECK-NEXT: stp q21, q20, [x8, #304] -; CHECK-NEXT: stp q17, q16, [x8, #368] -; CHECK-NEXT: stp q7, q5, [x8, #400] -; CHECK-NEXT: stp q4, q3, [x8, #432] -; CHECK-NEXT: stp q1, q2, [x8, #464] +; CHECK-NEXT: stp q6, q17, [x8, #368] +; CHECK-NEXT: stp q5, q4, [x8, #432] +; CHECK-NEXT: stp q2, q3, [x8, #464] ; CHECK-NEXT: str q0, [x8, #496] ; CHECK-NEXT: add sp, sp, #192 ; CHECK-NEXT: .cfi_def_cfa_offset 0 diff --git a/llvm/test/CodeGen/AArch64/rcpc3-sve.ll b/llvm/test/CodeGen/AArch64/rcpc3-sve.ll index 47e338151749..6b03e5d12bfd 100644 --- a/llvm/test/CodeGen/AArch64/rcpc3-sve.ll +++ b/llvm/test/CodeGen/AArch64/rcpc3-sve.ll @@ -8,8 +8,8 @@ define hidden @test_load_sve_lane0(ptr nocapture noundef readonly %a, noundef %b) local_unnamed_addr { ; CHECK-LABEL: test_load_sve_lane0: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl1 ; CHECK-NEXT: ldapr x8, [x0] +; CHECK-NEXT: ptrue p0.d, vl1 ; CHECK-NEXT: mov z0.d, p0/m, x8 ; CHECK-NEXT: ret %1 = load atomic i64, ptr %a acquire, align 8 @@ -20,9 +20,9 @@ define hidden @test_load_sve_lane0(ptr nocapture noundef read define hidden @test_load_sve_lane1(ptr nocapture noundef readonly %a, noundef %b) local_unnamed_addr { ; CHECK-LABEL: test_load_sve_lane1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, #1 // =0x1 ; CHECK-NEXT: index z1.d, #0, #1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z2.d, x8 ; CHECK-NEXT: ldapr x8, [x0] ; CHECK-NEXT: cmpeq p0.d, p0/z, z1.d, z2.d diff --git a/llvm/test/CodeGen/AArch64/reassocmls.ll b/llvm/test/CodeGen/AArch64/reassocmls.ll index 381caffba92e..acbf9fc584a2 100644 --- a/llvm/test/CodeGen/AArch64/reassocmls.ll +++ b/llvm/test/CodeGen/AArch64/reassocmls.ll @@ -79,7 +79,7 @@ define i64 @mls_i64_C(i64 %a, i64 %b, i64 %c, i64 %d, i64 %e) { ; CHECK-LABEL: mls_i64_C: ; CHECK: // %bb.0: ; CHECK-NEXT: mul x8, x2, x1 -; CHECK-NEXT: mov w9, #10 +; CHECK-NEXT: mov w9, #10 // =0xa ; CHECK-NEXT: madd x8, x4, x3, x8 ; CHECK-NEXT: sub x0, x9, x8 ; CHECK-NEXT: ret @@ -290,9 +290,9 @@ define @smlsl_nxv8i16( %a, @umlsl_nxv8i16( %a, %b, %c, %d, %e) { ; CHECK-LABEL: umlsl_nxv8i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: and z3.h, z3.h, #0xff ; CHECK-NEXT: and z4.h, z4.h, #0xff +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: and z1.h, z1.h, #0xff ; CHECK-NEXT: and z2.h, z2.h, #0xff ; CHECK-NEXT: mls z0.h, p0/m, z4.h, z3.h @@ -326,8 +326,8 @@ define @mls_nxv8i16( %a, define @mla_nxv8i16( %a, %b, %c, %d, %e) { ; CHECK-LABEL: mla_nxv8i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mul z1.h, z2.h, z1.h +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mla z1.h, p0/m, z4.h, z3.h ; CHECK-NEXT: add z0.h, z1.h, z0.h ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/reduce-shuffle.ll b/llvm/test/CodeGen/AArch64/reduce-shuffle.ll index a080a7403811..325ab444205b 100644 --- a/llvm/test/CodeGen/AArch64/reduce-shuffle.ll +++ b/llvm/test/CodeGen/AArch64/reduce-shuffle.ll @@ -20,111 +20,111 @@ define i32 @v1(ptr nocapture noundef readonly %p1, i32 noundef %i1, ptr nocaptur ; CHECK-NEXT: ldr d6, [x10, x8] ; CHECK-NEXT: ldr d5, [x11] ; CHECK-NEXT: ldr d7, [x11, x9] -; CHECK-NEXT: usubl v2.8h, v2.8b, v3.8b ; CHECK-NEXT: usubl v0.8h, v0.8b, v1.8b -; CHECK-NEXT: usubl v1.8h, v4.8b, v5.8b +; CHECK-NEXT: usubl v1.8h, v2.8b, v3.8b +; CHECK-NEXT: usubl v2.8h, v4.8b, v5.8b ; CHECK-NEXT: usubl v3.8h, v6.8b, v7.8b -; CHECK-NEXT: shll2 v4.4s, v2.8h, #16 -; CHECK-NEXT: shll2 v5.4s, v0.8h, #16 +; CHECK-NEXT: shll2 v4.4s, v0.8h, #16 +; CHECK-NEXT: shll2 v5.4s, v1.8h, #16 ; CHECK-NEXT: shll2 v6.4s, v3.8h, #16 -; CHECK-NEXT: shll2 v7.4s, v1.8h, #16 -; CHECK-NEXT: saddw v2.4s, v4.4s, v2.4h -; CHECK-NEXT: saddw v0.4s, v5.4s, v0.4h +; CHECK-NEXT: shll2 v7.4s, v2.8h, #16 +; CHECK-NEXT: saddw v0.4s, v4.4s, v0.4h +; CHECK-NEXT: saddw v1.4s, v5.4s, v1.4h ; CHECK-NEXT: saddw v3.4s, v6.4s, v3.4h -; CHECK-NEXT: saddw v1.4s, v7.4s, v1.4h +; CHECK-NEXT: saddw v2.4s, v7.4s, v2.4h +; CHECK-NEXT: zip1 v4.4s, v1.4s, v0.4s +; CHECK-NEXT: zip2 v6.4s, v1.4s, v0.4s +; CHECK-NEXT: uzp2 v5.4s, v3.4s, v2.4s ; CHECK-NEXT: mov v7.16b, v2.16b -; CHECK-NEXT: zip1 v4.4s, v2.4s, v0.4s -; CHECK-NEXT: zip2 v6.4s, v2.4s, v0.4s -; CHECK-NEXT: uzp2 v5.4s, v3.4s, v1.4s -; CHECK-NEXT: mov v17.16b, v1.16b -; CHECK-NEXT: zip2 v16.4s, v1.4s, v3.4s -; CHECK-NEXT: mov v7.s[3], v0.s[2] -; CHECK-NEXT: ext v18.16b, v3.16b, v3.16b, #12 -; CHECK-NEXT: ext v2.16b, v2.16b, v4.16b, #8 -; CHECK-NEXT: mov v17.s[1], v3.s[0] +; CHECK-NEXT: ext v17.16b, v3.16b, v3.16b, #12 +; CHECK-NEXT: zip2 v18.4s, v3.4s, v2.4s +; CHECK-NEXT: ext v16.16b, v1.16b, v4.16b, #8 +; CHECK-NEXT: mov v1.s[3], v0.s[2] +; CHECK-NEXT: mov v7.s[1], v3.s[0] ; CHECK-NEXT: uzp2 v0.4s, v5.4s, v3.4s -; CHECK-NEXT: zip2 v5.4s, v3.4s, v1.4s -; CHECK-NEXT: mov v3.s[0], v1.s[1] -; CHECK-NEXT: ext v1.16b, v1.16b, v18.16b, #12 -; CHECK-NEXT: mov v16.d[1], v7.d[1] -; CHECK-NEXT: mov v17.d[1], v2.d[1] +; CHECK-NEXT: zip2 v5.4s, v2.4s, v3.4s +; CHECK-NEXT: mov v3.s[0], v2.s[1] +; CHECK-NEXT: ext v2.16b, v2.16b, v17.16b, #12 +; CHECK-NEXT: mov v18.d[1], v1.d[1] +; CHECK-NEXT: mov v7.d[1], v16.d[1] ; CHECK-NEXT: mov v0.d[1], v6.d[1] -; CHECK-NEXT: mov v5.d[1], v7.d[1] ; CHECK-NEXT: mov v3.d[1], v4.d[1] -; CHECK-NEXT: mov v1.d[1], v6.d[1] -; CHECK-NEXT: add v0.4s, v0.4s, v5.4s -; CHECK-NEXT: add v2.4s, v3.4s, v17.4s -; CHECK-NEXT: sub v3.4s, v17.4s, v3.4s -; CHECK-NEXT: sub v1.4s, v16.4s, v1.4s +; CHECK-NEXT: mov v5.d[1], v1.d[1] +; CHECK-NEXT: mov v2.d[1], v6.d[1] +; CHECK-NEXT: add v0.4s, v0.4s, v18.4s +; CHECK-NEXT: add v1.4s, v3.4s, v7.4s +; CHECK-NEXT: sub v3.4s, v7.4s, v3.4s +; CHECK-NEXT: sub v2.4s, v5.4s, v2.4s ; CHECK-NEXT: rev64 v4.4s, v0.4s -; CHECK-NEXT: rev64 v5.4s, v2.4s -; CHECK-NEXT: add v6.4s, v1.4s, v3.4s -; CHECK-NEXT: sub v1.4s, v3.4s, v1.4s +; CHECK-NEXT: rev64 v6.4s, v1.4s +; CHECK-NEXT: sub v5.4s, v3.4s, v2.4s +; CHECK-NEXT: add v2.4s, v2.4s, v3.4s ; CHECK-NEXT: mov v4.d[1], v0.d[1] -; CHECK-NEXT: mov v5.d[1], v2.d[1] -; CHECK-NEXT: rev64 v3.4s, v1.4s -; CHECK-NEXT: sub v2.4s, v2.4s, v4.4s -; CHECK-NEXT: add v0.4s, v0.4s, v5.4s -; CHECK-NEXT: rev64 v4.4s, v6.4s -; CHECK-NEXT: rev64 v5.4s, v2.4s -; CHECK-NEXT: rev64 v7.4s, v0.4s -; CHECK-NEXT: addp v16.4s, v0.4s, v6.4s -; CHECK-NEXT: addp v17.4s, v2.4s, v1.4s -; CHECK-NEXT: sub v4.4s, v6.4s, v4.4s -; CHECK-NEXT: sub v1.4s, v1.4s, v3.4s -; CHECK-NEXT: sub v2.4s, v2.4s, v5.4s -; CHECK-NEXT: sub v0.4s, v0.4s, v7.4s -; CHECK-NEXT: zip1 v21.4s, v16.4s, v16.4s -; CHECK-NEXT: ext v5.16b, v17.16b, v1.16b, #4 -; CHECK-NEXT: ext v6.16b, v16.16b, v4.16b, #4 -; CHECK-NEXT: mov v18.16b, v1.16b -; CHECK-NEXT: mov v19.16b, v4.16b -; CHECK-NEXT: ext v3.16b, v2.16b, v17.16b, #8 -; CHECK-NEXT: ext v7.16b, v0.16b, v16.16b, #4 -; CHECK-NEXT: mov v18.s[2], v17.s[3] -; CHECK-NEXT: zip2 v5.4s, v5.4s, v17.4s -; CHECK-NEXT: zip2 v6.4s, v6.4s, v16.4s -; CHECK-NEXT: mov v19.s[2], v16.s[3] -; CHECK-NEXT: trn2 v0.4s, v21.4s, v0.4s -; CHECK-NEXT: ext v20.16b, v3.16b, v2.16b, #4 -; CHECK-NEXT: ext v7.16b, v7.16b, v7.16b, #4 -; CHECK-NEXT: mov v2.s[2], v17.s[1] -; CHECK-NEXT: ext v1.16b, v1.16b, v5.16b, #12 -; CHECK-NEXT: ext v4.16b, v4.16b, v6.16b, #12 -; CHECK-NEXT: mov v5.16b, v18.16b -; CHECK-NEXT: uzp2 v3.4s, v3.4s, v20.4s -; CHECK-NEXT: mov v6.16b, v7.16b -; CHECK-NEXT: mov v20.16b, v19.16b -; CHECK-NEXT: mov v21.16b, v2.16b -; CHECK-NEXT: mov v5.s[1], v17.s[2] -; CHECK-NEXT: sub v7.4s, v0.4s, v7.4s -; CHECK-NEXT: mov v6.s[0], v16.s[1] -; CHECK-NEXT: mov v20.s[1], v16.s[2] -; CHECK-NEXT: sub v16.4s, v19.4s, v4.4s -; CHECK-NEXT: mov v21.s[1], v17.s[0] -; CHECK-NEXT: sub v2.4s, v2.4s, v3.4s -; CHECK-NEXT: sub v17.4s, v18.4s, v1.4s -; CHECK-NEXT: add v1.4s, v5.4s, v1.4s +; CHECK-NEXT: mov v6.d[1], v1.d[1] +; CHECK-NEXT: rev64 v3.4s, v5.4s +; CHECK-NEXT: rev64 v7.4s, v2.4s +; CHECK-NEXT: sub v1.4s, v1.4s, v4.4s ; CHECK-NEXT: add v0.4s, v0.4s, v6.4s -; CHECK-NEXT: add v4.4s, v20.4s, v4.4s -; CHECK-NEXT: add v3.4s, v21.4s, v3.4s -; CHECK-NEXT: mov v1.d[1], v17.d[1] -; CHECK-NEXT: mov v0.d[1], v7.d[1] -; CHECK-NEXT: mov v4.d[1], v16.d[1] -; CHECK-NEXT: mov v3.d[1], v2.d[1] +; CHECK-NEXT: sub v3.4s, v5.4s, v3.4s +; CHECK-NEXT: addp v4.4s, v1.4s, v5.4s +; CHECK-NEXT: sub v5.4s, v2.4s, v7.4s +; CHECK-NEXT: addp v2.4s, v0.4s, v2.4s +; CHECK-NEXT: rev64 v6.4s, v0.4s +; CHECK-NEXT: rev64 v7.4s, v1.4s +; CHECK-NEXT: ext v16.16b, v4.16b, v3.16b, #4 +; CHECK-NEXT: ext v17.16b, v2.16b, v5.16b, #4 +; CHECK-NEXT: sub v0.4s, v0.4s, v6.4s +; CHECK-NEXT: sub v1.4s, v1.4s, v7.4s +; CHECK-NEXT: mov v7.16b, v3.16b +; CHECK-NEXT: zip2 v6.4s, v16.4s, v4.4s +; CHECK-NEXT: mov v16.16b, v5.16b +; CHECK-NEXT: zip2 v17.4s, v17.4s, v2.4s +; CHECK-NEXT: ext v18.16b, v0.16b, v2.16b, #4 +; CHECK-NEXT: mov v7.s[2], v4.s[3] +; CHECK-NEXT: mov v21.16b, v1.16b +; CHECK-NEXT: mov v16.s[2], v2.s[3] +; CHECK-NEXT: ext v5.16b, v5.16b, v17.16b, #12 +; CHECK-NEXT: zip1 v17.4s, v2.4s, v2.4s +; CHECK-NEXT: ext v3.16b, v3.16b, v6.16b, #12 +; CHECK-NEXT: ext v18.16b, v18.16b, v18.16b, #4 +; CHECK-NEXT: mov v19.16b, v7.16b +; CHECK-NEXT: ext v6.16b, v1.16b, v4.16b, #8 +; CHECK-NEXT: mov v21.s[2], v4.s[1] +; CHECK-NEXT: mov v20.16b, v16.16b +; CHECK-NEXT: mov v19.s[1], v4.s[2] +; CHECK-NEXT: trn2 v0.4s, v17.4s, v0.4s +; CHECK-NEXT: sub v16.4s, v16.4s, v5.4s +; CHECK-NEXT: mov v17.16b, v18.16b +; CHECK-NEXT: ext v1.16b, v6.16b, v1.16b, #4 +; CHECK-NEXT: sub v7.4s, v7.4s, v3.4s +; CHECK-NEXT: mov v20.s[1], v2.s[2] +; CHECK-NEXT: mov v17.s[0], v2.s[1] +; CHECK-NEXT: mov v2.16b, v21.16b +; CHECK-NEXT: add v3.4s, v19.4s, v3.4s +; CHECK-NEXT: uzp2 v1.4s, v6.4s, v1.4s +; CHECK-NEXT: add v5.4s, v20.4s, v5.4s +; CHECK-NEXT: mov v2.s[1], v4.s[0] +; CHECK-NEXT: sub v4.4s, v0.4s, v18.4s +; CHECK-NEXT: mov v3.d[1], v7.d[1] +; CHECK-NEXT: add v0.4s, v0.4s, v17.4s +; CHECK-NEXT: mov v5.d[1], v16.d[1] +; CHECK-NEXT: sub v6.4s, v21.4s, v1.4s +; CHECK-NEXT: add v1.4s, v2.4s, v1.4s +; CHECK-NEXT: mov v0.d[1], v4.d[1] +; CHECK-NEXT: cmlt v4.8h, v3.8h, #0 +; CHECK-NEXT: cmlt v2.8h, v5.8h, #0 +; CHECK-NEXT: mov v1.d[1], v6.d[1] +; CHECK-NEXT: add v3.4s, v4.4s, v3.4s +; CHECK-NEXT: cmlt v6.8h, v0.8h, #0 +; CHECK-NEXT: add v5.4s, v2.4s, v5.4s +; CHECK-NEXT: eor v3.16b, v3.16b, v4.16b ; CHECK-NEXT: cmlt v7.8h, v1.8h, #0 -; CHECK-NEXT: cmlt v2.8h, v0.8h, #0 -; CHECK-NEXT: cmlt v6.8h, v4.8h, #0 -; CHECK-NEXT: cmlt v5.8h, v3.8h, #0 +; CHECK-NEXT: add v0.4s, v6.4s, v0.4s +; CHECK-NEXT: eor v2.16b, v5.16b, v2.16b ; CHECK-NEXT: add v1.4s, v7.4s, v1.4s -; CHECK-NEXT: add v0.4s, v2.4s, v0.4s -; CHECK-NEXT: add v4.4s, v6.4s, v4.4s -; CHECK-NEXT: add v3.4s, v5.4s, v3.4s +; CHECK-NEXT: eor v0.16b, v0.16b, v6.16b +; CHECK-NEXT: add v2.4s, v2.4s, v3.4s ; CHECK-NEXT: eor v1.16b, v1.16b, v7.16b -; CHECK-NEXT: eor v0.16b, v0.16b, v2.16b -; CHECK-NEXT: eor v2.16b, v3.16b, v5.16b -; CHECK-NEXT: eor v3.16b, v4.16b, v6.16b -; CHECK-NEXT: add v1.4s, v3.4s, v1.4s ; CHECK-NEXT: add v0.4s, v0.4s, v2.4s ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: addv s0, v0.4s @@ -278,13 +278,13 @@ define i32 @v2(ptr nocapture noundef readonly %p1, i32 noundef %i1, ptr nocaptur ; CHECK-NEXT: mov v1.d[1], v6.d[1] ; CHECK-NEXT: add v2.4s, v2.4s, v16.4s ; CHECK-NEXT: add v3.4s, v4.4s, v17.4s +; CHECK-NEXT: rev64 v5.4s, v2.4s ; CHECK-NEXT: sub v0.4s, v0.4s, v1.4s ; CHECK-NEXT: sub v1.4s, v17.4s, v4.4s -; CHECK-NEXT: rev64 v5.4s, v2.4s ; CHECK-NEXT: rev64 v6.4s, v3.4s +; CHECK-NEXT: mov v5.d[1], v2.d[1] ; CHECK-NEXT: sub v4.4s, v1.4s, v0.4s ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s -; CHECK-NEXT: mov v5.d[1], v2.d[1] ; CHECK-NEXT: mov v6.d[1], v3.d[1] ; CHECK-NEXT: sub v3.4s, v3.4s, v5.4s ; CHECK-NEXT: add v1.4s, v2.4s, v6.4s @@ -304,40 +304,40 @@ define i32 @v2(ptr nocapture noundef readonly %p1, i32 noundef %i1, ptr nocaptur ; CHECK-NEXT: mov v4.d[1], v16.d[1] ; CHECK-NEXT: mov v1.d[1], v7.d[1] ; CHECK-NEXT: add v0.4s, v17.4s, v1.4s -; CHECK-NEXT: sub v1.4s, v1.4s, v17.4s ; CHECK-NEXT: add v2.4s, v18.4s, v4.4s +; CHECK-NEXT: sub v1.4s, v1.4s, v17.4s ; CHECK-NEXT: sub v3.4s, v4.4s, v18.4s -; CHECK-NEXT: zip2 v4.4s, v0.4s, v1.4s -; CHECK-NEXT: ext v5.16b, v0.16b, v0.16b, #4 -; CHECK-NEXT: ext v6.16b, v2.16b, v2.16b, #4 +; CHECK-NEXT: ext v4.16b, v0.16b, v0.16b, #4 +; CHECK-NEXT: ext v5.16b, v2.16b, v2.16b, #4 +; CHECK-NEXT: zip2 v6.4s, v0.4s, v1.4s ; CHECK-NEXT: zip2 v7.4s, v1.4s, v0.4s ; CHECK-NEXT: zip2 v16.4s, v3.4s, v2.4s ; CHECK-NEXT: zip2 v17.4s, v2.4s, v3.4s ; CHECK-NEXT: zip1 v0.4s, v0.4s, v1.4s -; CHECK-NEXT: zip1 v2.4s, v2.4s, v3.4s -; CHECK-NEXT: ext v1.16b, v5.16b, v1.16b, #8 -; CHECK-NEXT: ext v18.16b, v6.16b, v3.16b, #8 -; CHECK-NEXT: add v3.4s, v16.4s, v7.4s -; CHECK-NEXT: sub v4.4s, v4.4s, v17.4s -; CHECK-NEXT: sub v0.4s, v0.4s, v2.4s -; CHECK-NEXT: ext v1.16b, v1.16b, v5.16b, #4 -; CHECK-NEXT: ext v5.16b, v18.16b, v6.16b, #4 -; CHECK-NEXT: cmlt v2.8h, v4.8h, #0 -; CHECK-NEXT: cmlt v6.8h, v3.8h, #0 -; CHECK-NEXT: add v3.4s, v6.4s, v3.4s -; CHECK-NEXT: add v4.4s, v2.4s, v4.4s -; CHECK-NEXT: add v1.4s, v5.4s, v1.4s +; CHECK-NEXT: ext v18.16b, v4.16b, v1.16b, #8 +; CHECK-NEXT: ext v19.16b, v5.16b, v3.16b, #8 +; CHECK-NEXT: zip1 v1.4s, v2.4s, v3.4s +; CHECK-NEXT: add v2.4s, v16.4s, v7.4s +; CHECK-NEXT: sub v3.4s, v6.4s, v17.4s +; CHECK-NEXT: ext v4.16b, v18.16b, v4.16b, #4 +; CHECK-NEXT: ext v5.16b, v19.16b, v5.16b, #4 +; CHECK-NEXT: sub v0.4s, v0.4s, v1.4s +; CHECK-NEXT: cmlt v1.8h, v3.8h, #0 +; CHECK-NEXT: cmlt v6.8h, v2.8h, #0 +; CHECK-NEXT: add v4.4s, v5.4s, v4.4s ; CHECK-NEXT: cmlt v5.8h, v0.8h, #0 +; CHECK-NEXT: add v2.4s, v6.4s, v2.4s +; CHECK-NEXT: add v3.4s, v1.4s, v3.4s +; CHECK-NEXT: cmlt v7.8h, v4.8h, #0 ; CHECK-NEXT: add v0.4s, v5.4s, v0.4s -; CHECK-NEXT: eor v2.16b, v4.16b, v2.16b -; CHECK-NEXT: eor v3.16b, v3.16b, v6.16b -; CHECK-NEXT: cmlt v4.8h, v1.8h, #0 -; CHECK-NEXT: add v2.4s, v3.4s, v2.4s -; CHECK-NEXT: add v1.4s, v4.4s, v1.4s +; CHECK-NEXT: eor v2.16b, v2.16b, v6.16b +; CHECK-NEXT: eor v1.16b, v3.16b, v1.16b +; CHECK-NEXT: add v3.4s, v7.4s, v4.4s ; CHECK-NEXT: eor v0.16b, v0.16b, v5.16b -; CHECK-NEXT: add v0.4s, v0.4s, v2.4s -; CHECK-NEXT: eor v1.16b, v1.16b, v4.16b -; CHECK-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-NEXT: add v1.4s, v2.4s, v1.4s +; CHECK-NEXT: eor v2.16b, v3.16b, v7.16b +; CHECK-NEXT: add v0.4s, v0.4s, v1.4s +; CHECK-NEXT: add v0.4s, v2.4s, v0.4s ; CHECK-NEXT: addv s0, v0.4s ; CHECK-NEXT: fmov w8, s0 ; CHECK-NEXT: lsr w9, w8, #16 @@ -461,93 +461,93 @@ define i32 @v3(ptr nocapture noundef readonly %p1, i32 noundef %i1, ptr nocaptur ; CHECK-NEXT: ldr d3, [x11, x9] ; CHECK-NEXT: ldr d4, [x10] ; CHECK-NEXT: ldr d5, [x11] +; CHECK-NEXT: shll2 v6.4s, v0.8h, #16 ; CHECK-NEXT: usubl v2.8h, v2.8b, v3.8b -; CHECK-NEXT: usubl v3.8h, v4.8b, v5.8b -; CHECK-NEXT: shll2 v4.4s, v0.8h, #16 -; CHECK-NEXT: shll2 v5.4s, v1.8h, #16 -; CHECK-NEXT: saddw v0.4s, v4.4s, v0.4h -; CHECK-NEXT: shll2 v4.4s, v2.8h, #16 -; CHECK-NEXT: saddw v1.4s, v5.4s, v1.4h -; CHECK-NEXT: shll2 v5.4s, v3.8h, #16 -; CHECK-NEXT: saddw v2.4s, v4.4s, v2.4h -; CHECK-NEXT: saddw v3.4s, v5.4s, v3.4h -; CHECK-NEXT: rev64 v4.4s, v0.4s +; CHECK-NEXT: shll2 v3.4s, v1.8h, #16 +; CHECK-NEXT: usubl v4.8h, v4.8b, v5.8b +; CHECK-NEXT: saddw v0.4s, v6.4s, v0.4h +; CHECK-NEXT: shll2 v5.4s, v2.8h, #16 +; CHECK-NEXT: saddw v1.4s, v3.4s, v1.4h +; CHECK-NEXT: shll2 v3.4s, v4.8h, #16 +; CHECK-NEXT: rev64 v6.4s, v0.4s +; CHECK-NEXT: saddw v2.4s, v5.4s, v2.4h ; CHECK-NEXT: rev64 v5.4s, v1.4s -; CHECK-NEXT: rev64 v6.4s, v2.4s -; CHECK-NEXT: rev64 v7.4s, v3.4s -; CHECK-NEXT: sub v4.4s, v0.4s, v4.4s +; CHECK-NEXT: saddw v3.4s, v3.4s, v4.4h +; CHECK-NEXT: rev64 v4.4s, v2.4s +; CHECK-NEXT: sub v6.4s, v0.4s, v6.4s ; CHECK-NEXT: addp v0.4s, v1.4s, v0.4s +; CHECK-NEXT: rev64 v7.4s, v3.4s ; CHECK-NEXT: sub v5.4s, v1.4s, v5.4s -; CHECK-NEXT: sub v6.4s, v2.4s, v6.4s +; CHECK-NEXT: sub v4.4s, v2.4s, v4.4s ; CHECK-NEXT: addp v2.4s, v2.4s, v3.4s -; CHECK-NEXT: sub v1.4s, v3.4s, v7.4s +; CHECK-NEXT: ext v1.16b, v5.16b, v6.16b, #4 +; CHECK-NEXT: sub v7.4s, v3.4s, v7.4s ; CHECK-NEXT: ext v3.16b, v0.16b, v0.16b, #8 -; CHECK-NEXT: ext v7.16b, v5.16b, v4.16b, #4 -; CHECK-NEXT: mov v4.s[3], v5.s[2] -; CHECK-NEXT: zip2 v16.4s, v6.4s, v1.4s -; CHECK-NEXT: zip1 v1.4s, v6.4s, v1.4s -; CHECK-NEXT: uzp2 v6.4s, v2.4s, v0.4s -; CHECK-NEXT: ext v5.16b, v7.16b, v5.16b, #4 +; CHECK-NEXT: mov v6.s[3], v5.s[2] +; CHECK-NEXT: zip2 v16.4s, v4.4s, v7.4s +; CHECK-NEXT: zip1 v4.4s, v4.4s, v7.4s +; CHECK-NEXT: ext v1.16b, v1.16b, v5.16b, #4 +; CHECK-NEXT: uzp2 v5.4s, v2.4s, v0.4s ; CHECK-NEXT: uzp1 v0.4s, v2.4s, v0.4s ; CHECK-NEXT: uzp1 v7.4s, v2.4s, v3.4s ; CHECK-NEXT: uzp2 v2.4s, v2.4s, v3.4s -; CHECK-NEXT: mov v16.d[1], v4.d[1] -; CHECK-NEXT: rev64 v3.4s, v6.4s -; CHECK-NEXT: mov v1.d[1], v5.d[1] +; CHECK-NEXT: mov v16.d[1], v6.d[1] +; CHECK-NEXT: mov v4.d[1], v1.d[1] +; CHECK-NEXT: rev64 v1.4s, v5.4s ; CHECK-NEXT: rev64 v0.4s, v0.4s ; CHECK-NEXT: sub v2.4s, v7.4s, v2.4s -; CHECK-NEXT: sub v4.4s, v1.4s, v16.4s -; CHECK-NEXT: add v0.4s, v3.4s, v0.4s -; CHECK-NEXT: add v1.4s, v16.4s, v1.4s -; CHECK-NEXT: zip1 v3.4s, v2.4s, v4.4s +; CHECK-NEXT: sub v3.4s, v4.4s, v16.4s +; CHECK-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-NEXT: add v1.4s, v16.4s, v4.4s +; CHECK-NEXT: zip1 v4.4s, v2.4s, v3.4s ; CHECK-NEXT: zip1 v5.4s, v0.4s, v1.4s ; CHECK-NEXT: uzp2 v6.4s, v0.4s, v1.4s ; CHECK-NEXT: zip2 v7.4s, v0.4s, v1.4s -; CHECK-NEXT: zip2 v17.4s, v2.4s, v4.4s -; CHECK-NEXT: ext v16.16b, v2.16b, v3.16b, #8 +; CHECK-NEXT: zip2 v17.4s, v2.4s, v3.4s +; CHECK-NEXT: ext v16.16b, v2.16b, v4.16b, #8 ; CHECK-NEXT: trn2 v5.4s, v0.4s, v5.4s ; CHECK-NEXT: uzp2 v6.4s, v6.4s, v0.4s -; CHECK-NEXT: mov v2.s[3], v4.s[2] +; CHECK-NEXT: mov v2.s[3], v3.s[2] ; CHECK-NEXT: mov v0.s[1], v1.s[1] ; CHECK-NEXT: mov v5.d[1], v16.d[1] ; CHECK-NEXT: mov v6.d[1], v17.d[1] ; CHECK-NEXT: mov v7.d[1], v2.d[1] -; CHECK-NEXT: mov v0.d[1], v3.d[1] +; CHECK-NEXT: mov v0.d[1], v4.d[1] ; CHECK-NEXT: add v1.4s, v6.4s, v7.4s -; CHECK-NEXT: sub v2.4s, v7.4s, v6.4s -; CHECK-NEXT: add v3.4s, v5.4s, v0.4s +; CHECK-NEXT: add v2.4s, v5.4s, v0.4s +; CHECK-NEXT: sub v3.4s, v7.4s, v6.4s ; CHECK-NEXT: sub v0.4s, v0.4s, v5.4s -; CHECK-NEXT: zip2 v4.4s, v1.4s, v2.4s -; CHECK-NEXT: ext v5.16b, v1.16b, v1.16b, #4 -; CHECK-NEXT: ext v6.16b, v3.16b, v3.16b, #4 -; CHECK-NEXT: zip2 v7.4s, v2.4s, v1.4s -; CHECK-NEXT: zip2 v16.4s, v0.4s, v3.4s -; CHECK-NEXT: zip2 v17.4s, v3.4s, v0.4s -; CHECK-NEXT: zip1 v1.4s, v1.4s, v2.4s -; CHECK-NEXT: ext v2.16b, v5.16b, v2.16b, #8 -; CHECK-NEXT: ext v18.16b, v6.16b, v0.16b, #8 -; CHECK-NEXT: zip1 v0.4s, v3.4s, v0.4s -; CHECK-NEXT: add v3.4s, v16.4s, v7.4s -; CHECK-NEXT: sub v4.4s, v4.4s, v17.4s -; CHECK-NEXT: ext v2.16b, v2.16b, v5.16b, #4 -; CHECK-NEXT: ext v5.16b, v18.16b, v6.16b, #4 +; CHECK-NEXT: ext v4.16b, v1.16b, v1.16b, #4 +; CHECK-NEXT: ext v5.16b, v2.16b, v2.16b, #4 +; CHECK-NEXT: zip2 v6.4s, v1.4s, v3.4s +; CHECK-NEXT: zip2 v7.4s, v3.4s, v1.4s +; CHECK-NEXT: zip2 v16.4s, v0.4s, v2.4s +; CHECK-NEXT: zip2 v17.4s, v2.4s, v0.4s +; CHECK-NEXT: zip1 v1.4s, v1.4s, v3.4s +; CHECK-NEXT: ext v18.16b, v4.16b, v3.16b, #8 +; CHECK-NEXT: ext v19.16b, v5.16b, v0.16b, #8 +; CHECK-NEXT: zip1 v0.4s, v2.4s, v0.4s +; CHECK-NEXT: add v2.4s, v16.4s, v7.4s +; CHECK-NEXT: sub v3.4s, v6.4s, v17.4s +; CHECK-NEXT: ext v4.16b, v18.16b, v4.16b, #4 +; CHECK-NEXT: ext v5.16b, v19.16b, v5.16b, #4 ; CHECK-NEXT: sub v0.4s, v1.4s, v0.4s -; CHECK-NEXT: cmlt v1.8h, v4.8h, #0 -; CHECK-NEXT: cmlt v6.8h, v3.8h, #0 -; CHECK-NEXT: add v3.4s, v6.4s, v3.4s -; CHECK-NEXT: add v4.4s, v1.4s, v4.4s -; CHECK-NEXT: add v2.4s, v5.4s, v2.4s +; CHECK-NEXT: cmlt v1.8h, v3.8h, #0 +; CHECK-NEXT: cmlt v6.8h, v2.8h, #0 +; CHECK-NEXT: add v4.4s, v5.4s, v4.4s ; CHECK-NEXT: cmlt v5.8h, v0.8h, #0 +; CHECK-NEXT: add v2.4s, v6.4s, v2.4s +; CHECK-NEXT: add v3.4s, v1.4s, v3.4s +; CHECK-NEXT: cmlt v7.8h, v4.8h, #0 ; CHECK-NEXT: add v0.4s, v5.4s, v0.4s -; CHECK-NEXT: eor v1.16b, v4.16b, v1.16b -; CHECK-NEXT: eor v3.16b, v3.16b, v6.16b -; CHECK-NEXT: cmlt v4.8h, v2.8h, #0 -; CHECK-NEXT: add v1.4s, v3.4s, v1.4s -; CHECK-NEXT: add v2.4s, v4.4s, v2.4s +; CHECK-NEXT: eor v2.16b, v2.16b, v6.16b +; CHECK-NEXT: eor v1.16b, v3.16b, v1.16b +; CHECK-NEXT: add v3.4s, v7.4s, v4.4s ; CHECK-NEXT: eor v0.16b, v0.16b, v5.16b +; CHECK-NEXT: add v1.4s, v2.4s, v1.4s +; CHECK-NEXT: eor v2.16b, v3.16b, v7.16b ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s -; CHECK-NEXT: eor v1.16b, v2.16b, v4.16b -; CHECK-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-NEXT: add v0.4s, v2.4s, v0.4s ; CHECK-NEXT: addv s0, v0.4s ; CHECK-NEXT: fmov w8, s0 ; CHECK-NEXT: lsr w9, w8, #16 diff --git a/llvm/test/CodeGen/AArch64/sat-add.ll b/llvm/test/CodeGen/AArch64/sat-add.ll index 86c224bee990..2deb19be2482 100644 --- a/llvm/test/CodeGen/AArch64/sat-add.ll +++ b/llvm/test/CodeGen/AArch64/sat-add.ll @@ -346,9 +346,9 @@ define <16 x i8> @unsigned_sat_constant_v16i8_using_min(<16 x i8> %x) { ; CHECK-LABEL: unsigned_sat_constant_v16i8_using_min: ; CHECK: // %bb.0: ; CHECK-NEXT: movi v1.16b, #213 +; CHECK-NEXT: movi v2.16b, #42 ; CHECK-NEXT: umin v0.16b, v0.16b, v1.16b -; CHECK-NEXT: movi v1.16b, #42 -; CHECK-NEXT: add v0.16b, v0.16b, v1.16b +; CHECK-NEXT: add v0.16b, v0.16b, v2.16b ; CHECK-NEXT: ret %c = icmp ult <16 x i8> %x, %s = select <16 x i1> %c, <16 x i8> %x, <16 x i8> @@ -384,9 +384,9 @@ define <8 x i16> @unsigned_sat_constant_v8i16_using_min(<8 x i16> %x) { ; CHECK-LABEL: unsigned_sat_constant_v8i16_using_min: ; CHECK: // %bb.0: ; CHECK-NEXT: mvni v1.8h, #42 +; CHECK-NEXT: movi v2.8h, #42 ; CHECK-NEXT: umin v0.8h, v0.8h, v1.8h -; CHECK-NEXT: movi v1.8h, #42 -; CHECK-NEXT: add v0.8h, v0.8h, v1.8h +; CHECK-NEXT: add v0.8h, v0.8h, v2.8h ; CHECK-NEXT: ret %c = icmp ult <8 x i16> %x, %s = select <8 x i1> %c, <8 x i16> %x, <8 x i16> diff --git a/llvm/test/CodeGen/AArch64/sext.ll b/llvm/test/CodeGen/AArch64/sext.ll index 3e0d5dd87509..5237a3491de9 100644 --- a/llvm/test/CodeGen/AArch64/sext.ll +++ b/llvm/test/CodeGen/AArch64/sext.ll @@ -245,15 +245,15 @@ define <3 x i32> @sext_v3i8_v3i32(<3 x i8> %a) { ; CHECK-GI-LABEL: sext_v3i8_v3i32: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: mov w8, #24 // =0x18 -; CHECK-GI-NEXT: fmov s1, w0 -; CHECK-GI-NEXT: fmov s0, w8 -; CHECK-GI-NEXT: mov v1.s[1], w1 -; CHECK-GI-NEXT: mov v0.s[1], w8 -; CHECK-GI-NEXT: mov v1.s[2], w2 -; CHECK-GI-NEXT: mov v0.s[2], w8 -; CHECK-GI-NEXT: neg v2.4s, v0.4s -; CHECK-GI-NEXT: ushl v0.4s, v1.4s, v0.4s -; CHECK-GI-NEXT: sshl v0.4s, v0.4s, v2.4s +; CHECK-GI-NEXT: fmov s0, w0 +; CHECK-GI-NEXT: fmov s1, w8 +; CHECK-GI-NEXT: mov v0.s[1], w1 +; CHECK-GI-NEXT: mov v1.s[1], w8 +; CHECK-GI-NEXT: mov v0.s[2], w2 +; CHECK-GI-NEXT: mov v1.s[2], w8 +; CHECK-GI-NEXT: ushl v0.4s, v0.4s, v1.4s +; CHECK-GI-NEXT: neg v1.4s, v1.4s +; CHECK-GI-NEXT: sshl v0.4s, v0.4s, v1.4s ; CHECK-GI-NEXT: ret entry: %c = sext <3 x i8> %a to <3 x i32> @@ -408,15 +408,15 @@ define <3 x i32> @sext_v3i10_v3i32(<3 x i10> %a) { ; CHECK-GI-LABEL: sext_v3i10_v3i32: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: mov w8, #22 // =0x16 -; CHECK-GI-NEXT: fmov s1, w0 -; CHECK-GI-NEXT: fmov s0, w8 -; CHECK-GI-NEXT: mov v1.s[1], w1 -; CHECK-GI-NEXT: mov v0.s[1], w8 -; CHECK-GI-NEXT: mov v1.s[2], w2 -; CHECK-GI-NEXT: mov v0.s[2], w8 -; CHECK-GI-NEXT: neg v2.4s, v0.4s -; CHECK-GI-NEXT: ushl v0.4s, v1.4s, v0.4s -; CHECK-GI-NEXT: sshl v0.4s, v0.4s, v2.4s +; CHECK-GI-NEXT: fmov s0, w0 +; CHECK-GI-NEXT: fmov s1, w8 +; CHECK-GI-NEXT: mov v0.s[1], w1 +; CHECK-GI-NEXT: mov v1.s[1], w8 +; CHECK-GI-NEXT: mov v0.s[2], w2 +; CHECK-GI-NEXT: mov v1.s[2], w8 +; CHECK-GI-NEXT: ushl v0.4s, v0.4s, v1.4s +; CHECK-GI-NEXT: neg v1.4s, v1.4s +; CHECK-GI-NEXT: sshl v0.4s, v0.4s, v1.4s ; CHECK-GI-NEXT: ret entry: %c = sext <3 x i10> %a to <3 x i32> diff --git a/llvm/test/CodeGen/AArch64/sink-addsub-of-const.ll b/llvm/test/CodeGen/AArch64/sink-addsub-of-const.ll index f4f75bb9c782..88e062d2c999 100644 --- a/llvm/test/CodeGen/AArch64/sink-addsub-of-const.ll +++ b/llvm/test/CodeGen/AArch64/sink-addsub-of-const.ll @@ -158,8 +158,8 @@ define i32 @sink_sub_from_const_to_sub2(i32 %a, i32 %b) { define <4 x i32> @vec_sink_add_of_const_to_add0(<4 x i32> %a, <4 x i32> %b) { ; CHECK-LABEL: vec_sink_add_of_const_to_add0: ; CHECK: // %bb.0: -; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: adrp x8, .LCPI12_0 +; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI12_0] ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret @@ -170,8 +170,8 @@ define <4 x i32> @vec_sink_add_of_const_to_add0(<4 x i32> %a, <4 x i32> %b) { define <4 x i32> @vec_sink_add_of_const_to_add1(<4 x i32> %a, <4 x i32> %b) { ; CHECK-LABEL: vec_sink_add_of_const_to_add1: ; CHECK: // %bb.0: -; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: adrp x8, .LCPI13_0 +; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI13_0] ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret @@ -186,8 +186,8 @@ define <4 x i32> @vec_sink_add_of_const_to_add1(<4 x i32> %a, <4 x i32> %b) { define <4 x i32> @vec_sink_sub_of_const_to_add0(<4 x i32> %a, <4 x i32> %b) { ; CHECK-LABEL: vec_sink_sub_of_const_to_add0: ; CHECK: // %bb.0: -; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: adrp x8, .LCPI14_0 +; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI14_0] ; CHECK-NEXT: sub v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret @@ -198,8 +198,8 @@ define <4 x i32> @vec_sink_sub_of_const_to_add0(<4 x i32> %a, <4 x i32> %b) { define <4 x i32> @vec_sink_sub_of_const_to_add1(<4 x i32> %a, <4 x i32> %b) { ; CHECK-LABEL: vec_sink_sub_of_const_to_add1: ; CHECK: // %bb.0: -; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: adrp x8, .LCPI15_0 +; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI15_0] ; CHECK-NEXT: sub v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret @@ -214,8 +214,8 @@ define <4 x i32> @vec_sink_sub_of_const_to_add1(<4 x i32> %a, <4 x i32> %b) { define <4 x i32> @vec_sink_sub_from_const_to_add0(<4 x i32> %a, <4 x i32> %b) { ; CHECK-LABEL: vec_sink_sub_from_const_to_add0: ; CHECK: // %bb.0: -; CHECK-NEXT: sub v0.4s, v1.4s, v0.4s ; CHECK-NEXT: adrp x8, .LCPI16_0 +; CHECK-NEXT: sub v0.4s, v1.4s, v0.4s ; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI16_0] ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret @@ -226,8 +226,8 @@ define <4 x i32> @vec_sink_sub_from_const_to_add0(<4 x i32> %a, <4 x i32> %b) { define <4 x i32> @vec_sink_sub_from_const_to_add1(<4 x i32> %a, <4 x i32> %b) { ; CHECK-LABEL: vec_sink_sub_from_const_to_add1: ; CHECK: // %bb.0: -; CHECK-NEXT: sub v0.4s, v1.4s, v0.4s ; CHECK-NEXT: adrp x8, .LCPI17_0 +; CHECK-NEXT: sub v0.4s, v1.4s, v0.4s ; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI17_0] ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret @@ -242,8 +242,8 @@ define <4 x i32> @vec_sink_sub_from_const_to_add1(<4 x i32> %a, <4 x i32> %b) { define <4 x i32> @vec_sink_add_of_const_to_sub(<4 x i32> %a, <4 x i32> %b) { ; CHECK-LABEL: vec_sink_add_of_const_to_sub: ; CHECK: // %bb.0: -; CHECK-NEXT: sub v0.4s, v0.4s, v1.4s ; CHECK-NEXT: adrp x8, .LCPI18_0 +; CHECK-NEXT: sub v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI18_0] ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret @@ -254,8 +254,8 @@ define <4 x i32> @vec_sink_add_of_const_to_sub(<4 x i32> %a, <4 x i32> %b) { define <4 x i32> @vec_sink_add_of_const_to_sub2(<4 x i32> %a, <4 x i32> %b) { ; CHECK-LABEL: vec_sink_add_of_const_to_sub2: ; CHECK: // %bb.0: -; CHECK-NEXT: sub v0.4s, v1.4s, v0.4s ; CHECK-NEXT: adrp x8, .LCPI19_0 +; CHECK-NEXT: sub v0.4s, v1.4s, v0.4s ; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI19_0] ; CHECK-NEXT: sub v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret @@ -270,8 +270,8 @@ define <4 x i32> @vec_sink_add_of_const_to_sub2(<4 x i32> %a, <4 x i32> %b) { define <4 x i32> @vec_sink_sub_of_const_to_sub(<4 x i32> %a, <4 x i32> %b) { ; CHECK-LABEL: vec_sink_sub_of_const_to_sub: ; CHECK: // %bb.0: -; CHECK-NEXT: sub v0.4s, v0.4s, v1.4s ; CHECK-NEXT: adrp x8, .LCPI20_0 +; CHECK-NEXT: sub v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI20_0] ; CHECK-NEXT: sub v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret @@ -282,8 +282,8 @@ define <4 x i32> @vec_sink_sub_of_const_to_sub(<4 x i32> %a, <4 x i32> %b) { define <4 x i32> @vec_sink_sub_of_const_to_sub2(<4 x i32> %a, <4 x i32> %b) { ; CHECK-LABEL: vec_sink_sub_of_const_to_sub2: ; CHECK: // %bb.0: -; CHECK-NEXT: sub v0.4s, v1.4s, v0.4s ; CHECK-NEXT: adrp x8, .LCPI21_0 +; CHECK-NEXT: sub v0.4s, v1.4s, v0.4s ; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI21_0] ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret @@ -298,8 +298,8 @@ define <4 x i32> @vec_sink_sub_of_const_to_sub2(<4 x i32> %a, <4 x i32> %b) { define <4 x i32> @vec_sink_sub_from_const_to_sub(<4 x i32> %a, <4 x i32> %b) { ; CHECK-LABEL: vec_sink_sub_from_const_to_sub: ; CHECK: // %bb.0: -; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: adrp x8, .LCPI22_0 +; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI22_0] ; CHECK-NEXT: sub v0.4s, v1.4s, v0.4s ; CHECK-NEXT: ret @@ -310,8 +310,8 @@ define <4 x i32> @vec_sink_sub_from_const_to_sub(<4 x i32> %a, <4 x i32> %b) { define <4 x i32> @vec_sink_sub_from_const_to_sub2(<4 x i32> %a, <4 x i32> %b) { ; CHECK-LABEL: vec_sink_sub_from_const_to_sub2: ; CHECK: // %bb.0: -; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: adrp x8, .LCPI23_0 +; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI23_0] ; CHECK-NEXT: sub v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sme-pstate-sm-changing-call-disable-coalescing.ll b/llvm/test/CodeGen/AArch64/sme-pstate-sm-changing-call-disable-coalescing.ll index 0c674c5685e0..1d1bae42c9e3 100644 --- a/llvm/test/CodeGen/AArch64/sme-pstate-sm-changing-call-disable-coalescing.ll +++ b/llvm/test/CodeGen/AArch64/sme-pstate-sm-changing-call-disable-coalescing.ll @@ -28,8 +28,8 @@ define void @dont_coalesce_arg_i8(i8 %arg, ptr %ptr) #0 { ; CHECK-NEXT: smstop sm ; CHECK-NEXT: bl use_i8 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: ldr z0, [sp] // 16-byte Folded Reload +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: st1b { z0.b }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 ; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload @@ -61,8 +61,8 @@ define void @dont_coalesce_arg_i16(i16 %arg, ptr %ptr) #0 { ; CHECK-NEXT: smstop sm ; CHECK-NEXT: bl use_i16 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: ldr z0, [sp] // 16-byte Folded Reload +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: st1h { z0.h }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 ; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload @@ -94,8 +94,8 @@ define void @dont_coalesce_arg_i32(i32 %arg, ptr %ptr) #0 { ; CHECK-NEXT: smstop sm ; CHECK-NEXT: bl use_i32 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldr z0, [sp] // 16-byte Folded Reload +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: st1w { z0.s }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 ; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload @@ -127,8 +127,8 @@ define void @dont_coalesce_arg_i64(i64 %arg, ptr %ptr) #0 { ; CHECK-NEXT: smstop sm ; CHECK-NEXT: bl use_i64 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldr z0, [sp] // 16-byte Folded Reload +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: st1d { z0.d }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 ; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload @@ -165,8 +165,8 @@ define void @dont_coalesce_arg_f16(half %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr h0, [sp, #14] // 2-byte Folded Reload ; CHECK-NEXT: bl use_f16 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1h { z0.h }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -205,8 +205,8 @@ define void @dont_coalesce_arg_f32(float %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr s0, [sp, #12] // 4-byte Folded Reload ; CHECK-NEXT: bl use_f32 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1w { z0.s }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -245,8 +245,8 @@ define void @dont_coalesce_arg_f64(double %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr d0, [sp, #8] // 8-byte Folded Reload ; CHECK-NEXT: bl use_f64 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1d { z0.d }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -290,8 +290,8 @@ define void @dont_coalesce_arg_v1i8(<1 x i8> %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr d0, [sp, #8] // 8-byte Folded Reload ; CHECK-NEXT: bl use_v16i8 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1b { z0.b }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -331,8 +331,8 @@ define void @dont_coalesce_arg_v1i16(<1 x i16> %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr d0, [sp, #8] // 8-byte Folded Reload ; CHECK-NEXT: bl use_v8i16 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1h { z0.h }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -372,8 +372,8 @@ define void @dont_coalesce_arg_v1i32(<1 x i32> %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr d0, [sp, #8] // 8-byte Folded Reload ; CHECK-NEXT: bl use_v4i32 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1w { z0.s }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -413,8 +413,8 @@ define void @dont_coalesce_arg_v1i64(<1 x i64> %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr d0, [sp, #8] // 8-byte Folded Reload ; CHECK-NEXT: bl use_v2i64 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1d { z0.d }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -454,8 +454,8 @@ define void @dont_coalesce_arg_v1f16(<1 x half> %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr h0, [sp, #14] // 2-byte Folded Reload ; CHECK-NEXT: bl use_v8f16 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1h { z0.h }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -495,8 +495,8 @@ define void @dont_coalesce_arg_v1f32(<1 x float> %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr d0, [sp, #8] // 8-byte Folded Reload ; CHECK-NEXT: bl use_v4f32 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1w { z0.s }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -536,8 +536,8 @@ define void @dont_coalesce_arg_v1f64(<1 x double> %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr d0, [sp, #8] // 8-byte Folded Reload ; CHECK-NEXT: bl use_v2f64 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1d { z0.d }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -581,8 +581,8 @@ define void @dont_coalesce_arg_v16i8(<16 x i8> %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: bl use_v16i8 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1b { z0.b }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -621,8 +621,8 @@ define void @dont_coalesce_arg_v8i16(<8 x i16> %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: bl use_v8i16 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1h { z0.h }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -661,8 +661,8 @@ define void @dont_coalesce_arg_v4i32(<4 x i32> %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: bl use_v4i32 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1w { z0.s }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -701,8 +701,8 @@ define void @dont_coalesce_arg_v2i64(<2 x i64> %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: bl use_v2i64 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1d { z0.d }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -741,8 +741,8 @@ define void @dont_coalesce_arg_v8f16(<8 x half> %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: bl use_v8f16 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1h { z0.h }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -781,8 +781,8 @@ define void @dont_coalesce_arg_v8bf16(<8 x bfloat> %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: bl use_v8bf16 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1h { z0.h }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -821,8 +821,8 @@ define void @dont_coalesce_arg_v4f32(<4 x float> %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: bl use_v4f32 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1d { z0.d }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -861,8 +861,8 @@ define void @dont_coalesce_arg_v2f64(<2 x double> %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: bl use_v2f64 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1d { z0.d }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -894,9 +894,9 @@ define void @dont_coalesce_arg_v8i1(<8 x i1> %arg, ptr %ptr) #0 { ; CHECK-NEXT: stp x30, x19, [sp, #80] // 16-byte Folded Spill ; CHECK-NEXT: sub sp, sp, #16 ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: mov z1.d, z0.d +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: add x8, sp, #16 ; CHECK-NEXT: mov x19, x0 ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 @@ -942,13 +942,13 @@ define void @dont_coalesce_res_i8(ptr %ptr) #0 { ; CHECK-NEXT: smstop sm ; CHECK-NEXT: bl get_i8 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: fmov s0, w0 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload -; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload ; CHECK-NEXT: st1b { z0.b }, p0, [x19] ; CHECK-NEXT: ldp x30, x19, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp], #80 // 16-byte Folded Reload ; CHECK-NEXT: ret %res = call i8 @get_i8() @@ -969,13 +969,13 @@ define void @dont_coalesce_res_i16(ptr %ptr) #0 { ; CHECK-NEXT: smstop sm ; CHECK-NEXT: bl get_i16 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: fmov s0, w0 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload -; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload ; CHECK-NEXT: st1h { z0.h }, p0, [x19] ; CHECK-NEXT: ldp x30, x19, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp], #80 // 16-byte Folded Reload ; CHECK-NEXT: ret %res = call i16 @get_i16() @@ -996,13 +996,13 @@ define void @dont_coalesce_res_i32(ptr %ptr) #0 { ; CHECK-NEXT: smstop sm ; CHECK-NEXT: bl get_i32 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: fmov s0, w0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload -; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload ; CHECK-NEXT: st1w { z0.s }, p0, [x19] ; CHECK-NEXT: ldp x30, x19, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp], #80 // 16-byte Folded Reload ; CHECK-NEXT: ret %res = call i32 @get_i32() @@ -1023,13 +1023,13 @@ define void @dont_coalesce_res_i64(ptr %ptr) #0 { ; CHECK-NEXT: smstop sm ; CHECK-NEXT: bl get_i64 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: fmov d0, x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload -; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload ; CHECK-NEXT: st1d { z0.d }, p0, [x19] ; CHECK-NEXT: ldp x30, x19, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp], #80 // 16-byte Folded Reload ; CHECK-NEXT: ret %res = call i64 @get_i64() @@ -1056,11 +1056,11 @@ define void @dont_coalesce_res_f16(ptr %ptr) #0 { ; CHECK-NEXT: ldr h0, [sp, #14] // 2-byte Folded Reload ; CHECK-NEXT: // kill: def $h0 killed $h0 def $z0 ; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: st1h { z0.h }, p0, [x19] +; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload -; CHECK-NEXT: st1h { z0.h }, p0, [x19] -; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #96 ; CHECK-NEXT: ret %res = call half @get_f16() @@ -1086,11 +1086,11 @@ define void @dont_coalesce_res_f32(ptr %ptr) #0 { ; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldr s0, [sp, #12] // 4-byte Folded Reload ; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: st1w { z0.s }, p0, [x19] +; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload -; CHECK-NEXT: st1w { z0.s }, p0, [x19] -; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #96 ; CHECK-NEXT: ret %res = call float @get_f32() @@ -1116,11 +1116,11 @@ define void @dont_coalesce_res_f64(ptr %ptr) #0 { ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldr d0, [sp, #8] // 8-byte Folded Reload ; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: st1d { z0.d }, p0, [x19] +; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload -; CHECK-NEXT: st1d { z0.d }, p0, [x19] -; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #96 ; CHECK-NEXT: ret %res = call double @get_f64() @@ -1150,11 +1150,11 @@ define void @dont_coalesce_res_v1i8(ptr %ptr) #0 { ; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: ldr d0, [sp, #8] // 8-byte Folded Reload ; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: st1b { z0.b }, p0, [x19] +; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload -; CHECK-NEXT: st1b { z0.b }, p0, [x19] -; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #96 ; CHECK-NEXT: ret %res = call <1 x i8> @get_v1i8() @@ -1181,11 +1181,11 @@ define void @dont_coalesce_res_v1i16(ptr %ptr) #0 { ; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: ldr d0, [sp, #8] // 8-byte Folded Reload ; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: st1h { z0.h }, p0, [x19] +; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload -; CHECK-NEXT: st1h { z0.h }, p0, [x19] -; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #96 ; CHECK-NEXT: ret %res = call <1 x i16> @get_v1i16() @@ -1212,11 +1212,11 @@ define void @dont_coalesce_res_v1i32(ptr %ptr) #0 { ; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldr d0, [sp, #8] // 8-byte Folded Reload ; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: st1w { z0.s }, p0, [x19] +; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload -; CHECK-NEXT: st1w { z0.s }, p0, [x19] -; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #96 ; CHECK-NEXT: ret %res = call <1 x i32> @get_v1i32() @@ -1243,11 +1243,11 @@ define void @dont_coalesce_res_v1i64(ptr %ptr) #0 { ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldr d0, [sp, #8] // 8-byte Folded Reload ; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: st1d { z0.d }, p0, [x19] +; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload -; CHECK-NEXT: st1d { z0.d }, p0, [x19] -; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #96 ; CHECK-NEXT: ret %res = call <1 x i64> @get_v1i64() @@ -1275,11 +1275,11 @@ define void @dont_coalesce_res_v1f16(ptr %ptr) #0 { ; CHECK-NEXT: ldr h0, [sp, #14] // 2-byte Folded Reload ; CHECK-NEXT: // kill: def $h0 killed $h0 def $z0 ; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: st1h { z0.h }, p0, [x19] +; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload -; CHECK-NEXT: st1h { z0.h }, p0, [x19] -; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #96 ; CHECK-NEXT: ret %res = call <1 x half> @get_v1f16() @@ -1306,11 +1306,11 @@ define void @dont_coalesce_res_v1f32(ptr %ptr) #0 { ; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldr d0, [sp, #8] // 8-byte Folded Reload ; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: st1w { z0.s }, p0, [x19] +; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload -; CHECK-NEXT: st1w { z0.s }, p0, [x19] -; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #96 ; CHECK-NEXT: ret %res = call <1 x float> @get_v1f32() @@ -1337,11 +1337,11 @@ define void @dont_coalesce_res_v1f64(ptr %ptr) #0 { ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldr d0, [sp, #8] // 8-byte Folded Reload ; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: st1d { z0.d }, p0, [x19] +; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload -; CHECK-NEXT: st1d { z0.d }, p0, [x19] -; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #96 ; CHECK-NEXT: ret %res = call <1 x double> @get_v1f64() @@ -1373,11 +1373,11 @@ define void @dont_coalesce_res_v16i8(ptr %ptr) #0 { ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: st1b { z0.b }, p0, [x19] +; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload -; CHECK-NEXT: st1b { z0.b }, p0, [x19] -; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #96 ; CHECK-NEXT: ret %res = call <16 x i8> @get_v16i8() @@ -1404,11 +1404,11 @@ define void @dont_coalesce_res_v8i16(ptr %ptr) #0 { ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: st1h { z0.h }, p0, [x19] +; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload -; CHECK-NEXT: st1h { z0.h }, p0, [x19] -; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #96 ; CHECK-NEXT: ret %res = call <8 x i16> @get_v8i16() @@ -1435,11 +1435,11 @@ define void @dont_coalesce_res_v4i32(ptr %ptr) #0 { ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: st1w { z0.s }, p0, [x19] +; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload -; CHECK-NEXT: st1w { z0.s }, p0, [x19] -; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #96 ; CHECK-NEXT: ret %res = call <4 x i32> @get_v4i32() @@ -1466,11 +1466,11 @@ define void @dont_coalesce_res_v2i64(ptr %ptr) #0 { ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: st1d { z0.d }, p0, [x19] +; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload -; CHECK-NEXT: st1d { z0.d }, p0, [x19] -; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #96 ; CHECK-NEXT: ret %res = call <2 x i64> @get_v2i64() @@ -1497,11 +1497,11 @@ define void @dont_coalesce_res_v8f16(ptr %ptr) #0 { ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: st1h { z0.h }, p0, [x19] +; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload -; CHECK-NEXT: st1h { z0.h }, p0, [x19] -; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #96 ; CHECK-NEXT: ret %res = call <8 x half> @get_v8f16() @@ -1528,11 +1528,11 @@ define void @dont_coalesce_res_v4f32(ptr %ptr) #0 { ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: st1w { z0.s }, p0, [x19] +; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload -; CHECK-NEXT: st1w { z0.s }, p0, [x19] -; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #96 ; CHECK-NEXT: ret %res = call <4 x float> @get_v4f32() @@ -1559,11 +1559,11 @@ define void @dont_coalesce_res_v2f64(ptr %ptr) #0 { ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: st1d { z0.d }, p0, [x19] +; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload -; CHECK-NEXT: st1d { z0.d }, p0, [x19] -; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #96 ; CHECK-NEXT: ret %res = call <2 x double> @get_v2f64() diff --git a/llvm/test/CodeGen/AArch64/sme-streaming-compatible-interface.ll b/llvm/test/CodeGen/AArch64/sme-streaming-compatible-interface.ll index 47b24290d3c8..1e16f140676b 100644 --- a/llvm/test/CodeGen/AArch64/sme-streaming-compatible-interface.ll +++ b/llvm/test/CodeGen/AArch64/sme-streaming-compatible-interface.ll @@ -151,8 +151,8 @@ define <2 x double> @streaming_compatible_with_neon_vectors(<2 x double> %arg) " ; CHECK-NEXT: // %bb.3: ; CHECK-NEXT: smstart sm ; CHECK-NEXT: .LBB4_4: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: ldr z1, [x8] // 16-byte Folded Reload ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-fp-dots.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-fp-dots.ll index 0097968b1171..b4fd5a2272e7 100644 --- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-fp-dots.ll +++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-fp-dots.ll @@ -26,16 +26,16 @@ define void @fdot_multi_za32_f16_vg1x2(i32 %slice, %unused, < define void @fdot_multi_za32_f16_vg1x4(i32 %slice, %unused, %zn0, %zn1, %zn2, %zn3, ; CHECK-LABEL: fdot_multi_za32_f16_vg1x4: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1h { z27.h }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: fdot za.s[w8, 0, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: fdot za.s[w8, 7, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: ret @@ -71,16 +71,16 @@ define void @bfdot_multi_za32_bf16_vg1x2(i32 %slice, %unused, define void @fdot_multi_za32_bf16_vg1x4(i32 %slice, %unused, %zn0, %zn1, %zn2, %zn3, ; CHECK-LABEL: fdot_multi_za32_bf16_vg1x4: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1h { z27.h }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: bfdot za.s[w8, 0, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: bfdot za.s[w8, 7, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-int-dots.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-int-dots.ll index 6d9860483711..e154a4df86ef 100644 --- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-int-dots.ll +++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-int-dots.ll @@ -26,16 +26,16 @@ define void @udot_multi_za32_u16_vg1x2(i32 %slice, %unused, < define void @udot_multi_za32_u16_vg1x4(i32 %slice, %unused, %zn0, %zn1, %zn2, %zn3, ; CHECK-LABEL: udot_multi_za32_u16_vg1x4: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1h { z27.h }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: udot za.s[w8, 0, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: udot za.s[w8, 7, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: ret @@ -68,16 +68,16 @@ define void @udot_multi_za32_u8_vg1x2(i32 %slice, %unused, %unused, %zn0, %zn1, %zn2, %zn3, ; CHECK-LABEL: udot_multi_za32_u8_vg1x4: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1b { z27.b }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: udot za.s[w8, 0, vgx4], { z28.b - z31.b }, { z24.b - z27.b } ; CHECK-NEXT: udot za.s[w8, 7, vgx4], { z28.b - z31.b }, { z24.b - z27.b } ; CHECK-NEXT: ret @@ -110,16 +110,16 @@ define void @udot_multi_za64_u16_vg1x2(i32 %slice, %unused, < define void @udot_multi_za64_u16_vg1x4(i32 %slice, %unused, %zn0, %zn1, %zn2, %zn3, ; CHECK-LABEL: udot_multi_za64_u16_vg1x4: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1h { z27.h }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: udot za.d[w8, 0, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: udot za.d[w8, 7, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: ret @@ -152,16 +152,16 @@ define void @usdot_multi_za32_u8_vg1x2(i32 %slice, %unused, < define void @usdot_multi_za32_u8_vg1x4(i32 %slice, %unused, %zn0, %zn1, %zn2, %zn3, ; CHECK-LABEL: usdot_multi_za32_u8_vg1x4: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1b { z27.b }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: usdot za.s[w8, 0, vgx4], { z28.b - z31.b }, { z24.b - z27.b } ; CHECK-NEXT: usdot za.s[w8, 7, vgx4], { z28.b - z31.b }, { z24.b - z27.b } ; CHECK-NEXT: ret @@ -197,16 +197,16 @@ define void @sdot_multi_za32_u16_vg1x2(i32 %slice, %unused, < define void @sdot_multi_za32_u16_vg1x4(i32 %slice, %unused, %zn0, %zn1, %zn2, %zn3, ; CHECK-LABEL: sdot_multi_za32_u16_vg1x4: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1h { z27.h }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: sdot za.s[w8, 0, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: sdot za.s[w8, 7, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: ret @@ -239,16 +239,16 @@ define void @sdot_multi_za32_u8_vg1x2(i32 %slice, %unused, %unused, %zn0, %zn1, %zn2, %zn3, ; CHECK-LABEL: sdot_multi_za32_u8_vg1x4: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1b { z27.b }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: sdot za.s[w8, 0, vgx4], { z28.b - z31.b }, { z24.b - z27.b } ; CHECK-NEXT: sdot za.s[w8, 7, vgx4], { z28.b - z31.b }, { z24.b - z27.b } ; CHECK-NEXT: ret @@ -281,16 +281,16 @@ define void @sdot_multi_za64_u16_vg1x2(i32 %slice, %unused, < define void @sdot_multi_za64_u16_vg1x4(i32 %slice, %unused, %zn0, %zn1, %zn2, %zn3, ; CHECK-LABEL: sdot_multi_za64_u16_vg1x4: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1h { z27.h }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: sdot za.d[w8, 0, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: sdot za.d[w8, 7, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-max.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-max.ll index e95d29f65e55..92e8877927ea 100644 --- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-max.ll +++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-max.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -verify-machineinstrs < %s | FileCheck %s ; SMAX (Single, x2) @@ -151,8 +152,7 @@ define { , } @multi_vec_max_single_x2 ; SMAX (Single, x4) -define { , , , } -@multi_vec_max_single_x4_s8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_max_single_x4_s8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_max_single_x4_s8: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -170,8 +170,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_max_single_x4_s16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_max_single_x4_s16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_max_single_x4_s16: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -189,8 +188,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_max_single_x4_s32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_max_single_x4_s32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_max_single_x4_s32: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -208,8 +206,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_max_single_x4_s64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_max_single_x4_s64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_max_single_x4_s64: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -229,8 +226,7 @@ define { , , , , , , } -@multi_vec_max_single_x4_u8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_max_single_x4_u8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_max_single_x4_u8: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -248,8 +244,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_max_single_x4_u16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_max_single_x4_u16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_max_single_x4_u16: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -267,8 +262,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_max_single_x4_u32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_max_single_x4_u32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_max_single_x4_u32: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -286,8 +280,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_max_single_x4_u64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_max_single_x4_u64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_max_single_x4_u64: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -307,8 +300,7 @@ define { , , , , , , } -@multi_vec_max_single_x4_f16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_max_single_x4_f16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_max_single_x4_f16: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -326,8 +318,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_max_single_x4_f32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_max_single_x4_f32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_max_single_x4_f32: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -345,8 +336,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_max_single_x4_f64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_max_single_x4_f64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_max_single_x4_f64: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -537,104 +527,100 @@ define { , } @multi_vec_max_multi_x2_ ; SMAX (Multi, x4) -define { , , , } -@multi_vec_max_multi_x4_s8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_max_multi_x4_s8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_max_multi_x4_s8: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.b -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1b { z31.b }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: smax { z24.b - z27.b }, { z24.b - z27.b }, { z28.b - z31.b } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.smax.x4.nxv16i8( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_max_multi_x4_s16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_max_multi_x4_s16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_max_multi_x4_s16: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.h -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1h { z31.h }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: smax { z24.h - z27.h }, { z24.h - z27.h }, { z28.h - z31.h } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.smax.x4.nxv8i16( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_max_multi_x4_s32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_max_multi_x4_s32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_max_multi_x4_s32: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1w { z31.s }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: smax { z24.s - z27.s }, { z24.s - z27.s }, { z28.s - z31.s } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.smax.x4.nxv4i32( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_max_multi_x4_s64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_max_multi_x4_s64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_max_multi_x4_s64: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1d { z31.d }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: smax { z24.d - z27.d }, { z24.d - z27.d }, { z28.d - z31.d } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.smax.x4.nxv2i64( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) @@ -643,104 +629,100 @@ define { , , , , , , } -@multi_vec_max_multi_x4_u8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_max_multi_x4_u8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_max_multi_x4_u8: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.b -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1b { z31.b }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: umax { z24.b - z27.b }, { z24.b - z27.b }, { z28.b - z31.b } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.umax.x4.nxv16i8( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_max_multi_x4_u16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_max_multi_x4_u16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_max_multi_x4_u16: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.h -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1h { z31.h }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: umax { z24.h - z27.h }, { z24.h - z27.h }, { z28.h - z31.h } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.umax.x4.nxv8i16( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_max_multi_x4_u32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_max_multi_x4_u32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_max_multi_x4_u32: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1w { z31.s }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: umax { z24.s - z27.s }, { z24.s - z27.s }, { z28.s - z31.s } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.umax.x4.nxv4i32( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_max_multi_x4_u64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_max_multi_x4_u64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_max_multi_x4_u64: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1d { z31.d }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: umax { z24.d - z27.d }, { z24.d - z27.d }, { z28.d - z31.d } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.umax.x4.nxv2i64( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) @@ -749,78 +731,75 @@ define { , , , , , , } -@multi_vec_max_multi_x4_f16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_max_multi_x4_f16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_max_multi_x4_f16: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.h -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1h { z31.h }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: fmax { z24.h - z27.h }, { z24.h - z27.h }, { z28.h - z31.h } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.fmax.x4.nxv8f16( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_max_multi_x4_f32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_max_multi_x4_f32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_max_multi_x4_f32: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1w { z31.s }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: fmax { z24.s - z27.s }, { z24.s - z27.s }, { z28.s - z31.s } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.fmax.x4.nxv4f32( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_max_multi_x4_f64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_max_multi_x4_f64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_max_multi_x4_f64: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1d { z31.d }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: fmax { z24.d - z27.d }, { z24.d - z27.d }, { z28.d - z31.d } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.fmax.x4.nxv2f64( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) @@ -870,8 +849,7 @@ define { , } @multi_vec_maxnm_single ; FMAXNM (Single, x4) -define { , , , } -@multi_vec_maxnm_single_x4_f16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_maxnm_single_x4_f16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_maxnm_single_x4_f16: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -889,8 +867,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_maxnm_single_x4_f32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_maxnm_single_x4_f32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_maxnm_single_x4_f32: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -908,8 +885,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_maxnm_single_x4_f64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_maxnm_single_x4_f64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_maxnm_single_x4_f64: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -976,19 +952,18 @@ define { , } @multi_vec_maxnm_x2_f64( ; FMAXNM (Multi, x4) -define { , , , } -@multi_vec_maxnm_x4_f16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_maxnm_x4_f16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { ; CHECK-LABEL: multi_vec_maxnm_x4_f16: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.h -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1h { z31.h }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: fmaxnm { z24.h - z27.h }, { z24.h - z27.h }, { z28.h - z31.h } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d @@ -996,24 +971,23 @@ define { , , , , , , } - @llvm.aarch64.sve.fmaxnm.x4.nxv8f16( %zdn1, %zdn2, %zdn3, %zdn4, + @llvm.aarch64.sve.fmaxnm.x4.nxv8f16( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_maxnm_x4_f32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_maxnm_x4_f32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { ; CHECK-LABEL: multi_vec_maxnm_x4_f32: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1w { z31.s }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: fmaxnm { z24.s - z27.s }, { z24.s - z27.s }, { z28.s - z31.s } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d @@ -1026,19 +1000,18 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_maxnm_x4_f64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_maxnm_x4_f64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { ; CHECK-LABEL: multi_vec_maxnm_x4_f64: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1d { z31.d }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: fmaxnm { z24.d - z27.d }, { z24.d - z27.d }, { z28.d - z31.d } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-min.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-min.ll index 21a55c6436ac..363f9ba5d353 100644 --- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-min.ll +++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-min.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -verify-machineinstrs < %s | FileCheck %s ; SMIN (Single, x2) @@ -151,8 +152,7 @@ define { , } @multi_vec_min_single_x2 ; SMIN (Single, x4) -define { , , , } -@multi_vec_min_single_x4_s8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_min_single_x4_s8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_min_single_x4_s8: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -170,8 +170,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_min_single_x4_s16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_min_single_x4_s16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_min_single_x4_s16: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -189,8 +188,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_min_single_x4_s32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_min_single_x4_s32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_min_single_x4_s32: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -208,8 +206,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_min_single_x4_s64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_min_single_x4_s64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_min_single_x4_s64: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -229,8 +226,7 @@ define { , , , , , , } -@multi_vec_min_single_x4_u8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_min_single_x4_u8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_min_single_x4_u8: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -248,8 +244,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_min_single_x4_u16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_min_single_x4_u16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_min_single_x4_u16: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -267,8 +262,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_min_single_x4_u32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_min_single_x4_u32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_min_single_x4_u32: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -286,8 +280,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_min_single_x4_u64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_min_single_x4_u64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_min_single_x4_u64: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -307,8 +300,7 @@ define { , , , , , , } -@multi_vec_min_single_x4_f16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_min_single_x4_f16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_min_single_x4_f16: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -326,8 +318,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_min_single_x4_f32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_min_single_x4_f32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_min_single_x4_f32: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -345,8 +336,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_min_single_x4_f64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_min_single_x4_f64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_min_single_x4_f64: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -537,104 +527,100 @@ define { , } @multi_vec_min_multi_x2_ ; SMIN (Multi, x4) -define { , , , } -@multi_vec_min_multi_x4_s8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_min_multi_x4_s8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_min_multi_x4_s8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z30.d, z7.d ; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z29.d, z6.d ; CHECK-NEXT: mov z26.d, z3.d ; CHECK-NEXT: mov z28.d, z5.d ; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: ld1b { z31.b }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: smin { z24.b - z27.b }, { z24.b - z27.b }, { z28.b - z31.b } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.smin.x4.nxv16i8( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_min_multi_x4_s16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_min_multi_x4_s16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_min_multi_x4_s16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z30.d, z7.d ; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z29.d, z6.d ; CHECK-NEXT: mov z26.d, z3.d ; CHECK-NEXT: mov z28.d, z5.d ; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: ld1h { z31.h }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: smin { z24.h - z27.h }, { z24.h - z27.h }, { z28.h - z31.h } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.smin.x4.nxv8i16( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_min_multi_x4_s32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_min_multi_x4_s32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_min_multi_x4_s32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z30.d, z7.d ; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z29.d, z6.d ; CHECK-NEXT: mov z26.d, z3.d ; CHECK-NEXT: mov z28.d, z5.d ; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: ld1w { z31.s }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: smin { z24.s - z27.s }, { z24.s - z27.s }, { z28.s - z31.s } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.smin.x4.nxv4i32( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_min_multi_x4_s64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_min_multi_x4_s64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_min_multi_x4_s64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z30.d, z7.d ; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z29.d, z6.d ; CHECK-NEXT: mov z26.d, z3.d ; CHECK-NEXT: mov z28.d, z5.d ; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: ld1d { z31.d }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: smin { z24.d - z27.d }, { z24.d - z27.d }, { z28.d - z31.d } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.smin.x4.nxv2i64( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) @@ -643,104 +629,100 @@ define { , , , , , , } -@multi_vec_min_multi_x4_u8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_min_multi_x4_u8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_min_multi_x4_u8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z30.d, z7.d ; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z29.d, z6.d ; CHECK-NEXT: mov z26.d, z3.d ; CHECK-NEXT: mov z28.d, z5.d ; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: ld1b { z31.b }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: umin { z24.b - z27.b }, { z24.b - z27.b }, { z28.b - z31.b } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.umin.x4.nxv16i8( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_min_multi_x4_u16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_min_multi_x4_u16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_min_multi_x4_u16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z30.d, z7.d ; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z29.d, z6.d ; CHECK-NEXT: mov z26.d, z3.d ; CHECK-NEXT: mov z28.d, z5.d ; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: ld1h { z31.h }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: umin { z24.h - z27.h }, { z24.h - z27.h }, { z28.h - z31.h } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.umin.x4.nxv8i16( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_min_multi_x4_u32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_min_multi_x4_u32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_min_multi_x4_u32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z30.d, z7.d ; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z29.d, z6.d ; CHECK-NEXT: mov z26.d, z3.d ; CHECK-NEXT: mov z28.d, z5.d ; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: ld1w { z31.s }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: umin { z24.s - z27.s }, { z24.s - z27.s }, { z28.s - z31.s } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.umin.x4.nxv4i32( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_min_multi_x4_u64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_min_multi_x4_u64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_min_multi_x4_u64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z30.d, z7.d ; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z29.d, z6.d ; CHECK-NEXT: mov z26.d, z3.d ; CHECK-NEXT: mov z28.d, z5.d ; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: ld1d { z31.d }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: umin { z24.d - z27.d }, { z24.d - z27.d }, { z28.d - z31.d } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.umin.x4.nxv2i64( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) @@ -749,78 +731,75 @@ define { , , , , , , } -@multi_vec_min_multi_x4_f16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_min_multi_x4_f16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_min_multi_x4_f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z30.d, z7.d ; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z29.d, z6.d ; CHECK-NEXT: mov z26.d, z3.d ; CHECK-NEXT: mov z28.d, z5.d ; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: ld1h { z31.h }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: fmin { z24.h - z27.h }, { z24.h - z27.h }, { z28.h - z31.h } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.fmin.x4.nxv8f16( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_min_multi_x4_f32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_min_multi_x4_f32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_min_multi_x4_f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z30.d, z7.d ; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z29.d, z6.d ; CHECK-NEXT: mov z26.d, z3.d ; CHECK-NEXT: mov z28.d, z5.d ; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: ld1w { z31.s }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: fmin { z24.s - z27.s }, { z24.s - z27.s }, { z28.s - z31.s } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.fmin.x4.nxv4f32( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_min_multi_x4_f64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_min_multi_x4_f64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_min_multi_x4_f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z30.d, z7.d ; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z29.d, z6.d ; CHECK-NEXT: mov z26.d, z3.d ; CHECK-NEXT: mov z28.d, z5.d ; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: ld1d { z31.d }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: fmin { z24.d - z27.d }, { z24.d - z27.d }, { z28.d - z31.d } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.fmin.x4.nxv2f64( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) @@ -870,8 +849,7 @@ define { , } @multi_vec_minnm_single ; FMINNM (Single, x4) -define { , , , } -@multi_vec_minnm_single_x4_f16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_minnm_single_x4_f16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_minnm_single_x4_f16: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -889,8 +867,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_minnm_single_x4_f32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_minnm_single_x4_f32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_minnm_single_x4_f32: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -908,8 +885,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_minnm_single_x4_f64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_minnm_single_x4_f64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_minnm_single_x4_f64: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -976,19 +952,18 @@ define { , } @multi_vec_minnm_x2_f64( ; FMINNM (Multi, x4) -define { , , , } -@multi_vec_minnm_x4_f16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_minnm_x4_f16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { ; CHECK-LABEL: multi_vec_minnm_x4_f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z30.d, z7.d ; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z29.d, z6.d ; CHECK-NEXT: mov z26.d, z3.d ; CHECK-NEXT: mov z28.d, z5.d ; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: ld1h { z31.h }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: fminnm { z24.h - z27.h }, { z24.h - z27.h }, { z28.h - z31.h } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d @@ -1001,19 +976,18 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_minnm_x4_f32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_minnm_x4_f32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { ; CHECK-LABEL: multi_vec_minnm_x4_f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z30.d, z7.d ; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z29.d, z6.d ; CHECK-NEXT: mov z26.d, z3.d ; CHECK-NEXT: mov z28.d, z5.d ; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: ld1w { z31.s }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: fminnm { z24.s - z27.s }, { z24.s - z27.s }, { z28.s - z31.s } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d @@ -1026,19 +1000,18 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_minnm_x4_f64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_minnm_x4_f64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { ; CHECK-LABEL: multi_vec_minnm_x4_f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z30.d, z7.d ; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z29.d, z6.d ; CHECK-NEXT: mov z26.d, z3.d ; CHECK-NEXT: mov z28.d, z5.d ; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: ld1d { z31.d }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: fminnm { z24.d - z27.d }, { z24.d - z27.d }, { z28.d - z31.d } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-mlall.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-mlall.ll index f766bfcff4d1..346afc611eb7 100644 --- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-mlall.ll +++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-mlall.ll @@ -142,16 +142,16 @@ define void @multi_vector_mul_add_multi_long_vg4x2_s16(i32 %slice, %dummy, %zn0, %zn1, %zn2, %zn3, %zm0, %zm1, %zm2, %zm3) { ; CHECK-LABEL: multi_vector_mul_add_multi_long_vg4x4_s8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1b { z27.b }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: smlall za.s[w8, 0:3, vgx4], { z28.b - z31.b }, { z24.b - z27.b } ; CHECK-NEXT: smlall za.s[w8, 4:7, vgx4], { z28.b - z31.b }, { z24.b - z27.b } ; CHECK-NEXT: ret @@ -164,16 +164,16 @@ define void @multi_vector_mul_add_multi_long_vg4x4_s8(i32 %slice, %dummy, %zn0, %zn1, %zn2, %zn3, %zm0, %zm1, %zm2, %zm3) { ; CHECK-LABEL: multi_vector_mul_add_multi_long_vg4x4_s16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1h { z27.h }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: smlall za.d[w8, 0:3, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: smlall za.d[w8, 4:7, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: ret @@ -418,16 +418,16 @@ define void @multi_vector_mul_add_multi_long_vg4x2_u16(i32 %slice, %dummy, %zn0, %zn1, %zn2, %zn3, %zm0, %zm1, %zm2, %zm3) { ; CHECK-LABEL: multi_vector_mul_add_multi_long_vg4x4_u8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1b { z27.b }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: umlall za.s[w8, 0:3, vgx4], { z28.b - z31.b }, { z24.b - z27.b } ; CHECK-NEXT: umlall za.s[w8, 4:7, vgx4], { z28.b - z31.b }, { z24.b - z27.b } ; CHECK-NEXT: ret @@ -440,16 +440,16 @@ define void @multi_vector_mul_add_multi_long_vg4x4_u8(i32 %slice, %dummy, %zn0, %zn1, %zn2, %zn3, %zm0, %zm1, %zm2, %zm3) { ; CHECK-LABEL: multi_vector_mul_add_multi_long_vg4x4_u16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1h { z27.h }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: umlall za.d[w8, 0:3, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: umlall za.d[w8, 4:7, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: ret @@ -694,16 +694,16 @@ define void @multi_vector_mul_sub_multi_long_vg4x2_s16(i32 %slice, %dummy, %zn0, %zn1, %zn2, %zn3, %zm0, %zm1, %zm2, %zm3) { ; CHECK-LABEL: multi_vector_mul_sub_multi_long_vg4x4_s8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1b { z27.b }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: smlsll za.s[w8, 0:3, vgx4], { z28.b - z31.b }, { z24.b - z27.b } ; CHECK-NEXT: smlsll za.s[w8, 4:7, vgx4], { z28.b - z31.b }, { z24.b - z27.b } ; CHECK-NEXT: ret @@ -716,16 +716,16 @@ define void @multi_vector_mul_sub_multi_long_vg4x4_s8(i32 %slice, %dummy, %zn0, %zn1, %zn2, %zn3, %zm0, %zm1, %zm2, %zm3) { ; CHECK-LABEL: multi_vector_mul_sub_multi_long_vg4x4_s16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1h { z27.h }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: smlsll za.d[w8, 0:3, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: smlsll za.d[w8, 4:7, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: ret @@ -970,16 +970,16 @@ define void @multi_vector_mul_sub_multi_long_vg4x2_u16(i32 %slice, %dummy, %zn0, %zn1, %zn2, %zn3, %zm0, %zm1, %zm2, %zm3) { ; CHECK-LABEL: multi_vector_mul_sub_multi_long_vg4x4_u8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1b { z27.b }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: umlsll za.s[w8, 0:3, vgx4], { z28.b - z31.b }, { z24.b - z27.b } ; CHECK-NEXT: umlsll za.s[w8, 4:7, vgx4], { z28.b - z31.b }, { z24.b - z27.b } ; CHECK-NEXT: ret @@ -992,16 +992,16 @@ define void @multi_vector_mul_sub_multi_long_vg4x4_u8(i32 %slice, %dummy, %zn0, %zn1, %zn2, %zn3, %zm0, %zm1, %zm2, %zm3) { ; CHECK-LABEL: multi_vector_mul_sub_multi_long_vg4x4_u16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1h { z27.h }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: umlsll za.d[w8, 0:3, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: umlsll za.d[w8, 4:7, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: ret @@ -1275,16 +1275,16 @@ define void @multi_vector_mul_add_multi_unsigned_long_vg4x2_u8(i32 %slice, %dummy, %zn0, %zn1, %zn2, %zn3, %zm0, %zm1, %zm2, %zm3) { ; CHECK-LABEL: multi_vector_mul_add_multi_unsigned_long_vg4x4_u8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1b { z27.b }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: usmlall za.s[w8, 0:3, vgx4], { z28.b - z31.b }, { z24.b - z27.b } ; CHECK-NEXT: usmlall za.s[w8, 4:7, vgx4], { z28.b - z31.b }, { z24.b - z27.b } ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-rshl.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-rshl.ll index d138a3af4385..12a940ff03e2 100644 --- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-rshl.ll +++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-rshl.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -verify-machineinstrs < %s | FileCheck %s ; SRSHL (Single, x2) @@ -56,8 +57,7 @@ define { , } @multi_vec_rounding_shl_single ; SRSHL (Single, x4) -define { , , , } -@multi_vec_rounding_shl_single_x4_s8( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_rounding_shl_single_x4_s8( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_rounding_shl_single_x4_s8: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -75,8 +75,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_rounding_shl_single_x4_s16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_rounding_shl_single_x4_s16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_rounding_shl_single_x4_s16: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -94,8 +93,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_rounding_shl_single_x4_s32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_rounding_shl_single_x4_s32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_rounding_shl_single_x4_s32: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -113,8 +111,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_rounding_shl_single_x4_s64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_rounding_shl_single_x4_s64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_rounding_shl_single_x4_s64: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -188,8 +185,7 @@ define { , } @multi_vec_rounding_shl_single ; URSHL (Single, x4) -define { , , , } -@multi_vec_rounding_shl_single_x4_u8( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_rounding_shl_single_x4_u8( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_rounding_shl_single_x4_u8: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -207,8 +203,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_rounding_shl_single_x4_u16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_rounding_shl_single_x4_u16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_rounding_shl_single_x4_u16: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -226,8 +221,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_rounding_shl_single_x4_u32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_rounding_shl_single_x4_u32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_rounding_shl_single_x4_u32: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -245,8 +239,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_rounding_shl_single_x4_u64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_rounding_shl_single_x4_u64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_rounding_shl_single_x4_u64: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -328,19 +321,18 @@ define { , } @multi_vec_rounding_shl_x2_s64 ; SRSHL (Multi, x4) -define { , , , } -@multi_vec_rounding_shl_x4_s8( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_rounding_shl_x4_s8( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { ; CHECK-LABEL: multi_vec_rounding_shl_x4_s8: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.b -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1b { z31.b }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: srshl { z24.b - z27.b }, { z24.b - z27.b }, { z28.b - z31.b } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d @@ -353,19 +345,18 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_rounding_shl_x4_s16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_rounding_shl_x4_s16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { ; CHECK-LABEL: multi_vec_rounding_shl_x4_s16: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.h -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1h { z31.h }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: srshl { z24.h - z27.h }, { z24.h - z27.h }, { z28.h - z31.h } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d @@ -378,19 +369,18 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_rounding_shl_x4_s32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_rounding_shl_x4_s32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { ; CHECK-LABEL: multi_vec_rounding_shl_x4_s32: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1w { z31.s }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: srshl { z24.s - z27.s }, { z24.s - z27.s }, { z28.s - z31.s } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d @@ -403,19 +393,18 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_rounding_shl_x4_s64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_rounding_shl_x4_s64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { ; CHECK-LABEL: multi_vec_rounding_shl_x4_s64: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1d { z31.d }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: srshl { z24.d - z27.d }, { z24.d - z27.d }, { z28.d - z31.d } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d @@ -492,19 +481,18 @@ define { , } @multi_vec_rounding_uhl_x2_u64 ; URSHL (Multi, x4) -define { , , , } -@multi_vec_rounding_shl_x4_u8( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_rounding_shl_x4_u8( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { ; CHECK-LABEL: multi_vec_rounding_shl_x4_u8: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.b -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1b { z31.b }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: urshl { z24.b - z27.b }, { z24.b - z27.b }, { z28.b - z31.b } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d @@ -517,19 +505,18 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_rounding_shl_x4_u16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_rounding_shl_x4_u16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { ; CHECK-LABEL: multi_vec_rounding_shl_x4_u16: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.h -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1h { z31.h }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: urshl { z24.h - z27.h }, { z24.h - z27.h }, { z28.h - z31.h } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d @@ -542,19 +529,18 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_rounding_shl_x4_u32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_rounding_shl_x4_u32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { ; CHECK-LABEL: multi_vec_rounding_shl_x4_u32: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1w { z31.s }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: urshl { z24.s - z27.s }, { z24.s - z27.s }, { z28.s - z31.s } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d @@ -567,19 +553,18 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_rounding_shl_x4_u64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_rounding_shl_x4_u64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { ; CHECK-LABEL: multi_vec_rounding_shl_x4_u64: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1d { z31.d }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: urshl { z24.d - z27.d }, { z24.d - z27.d }, { z28.d - z31.d } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-sqdmulh.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-sqdmulh.ll index 9c5dff6c3bf6..e71afe213d8a 100644 --- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-sqdmulh.ll +++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-sqdmulh.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -verify-machineinstrs < %s | FileCheck %s ; SQDMULH (Single, x2) @@ -56,8 +57,7 @@ define { , } @multi_vec_sat_double_mulh_sin ; SQDMULH (Single, x4) -define { , , , } -@multi_vec_sat_double_mulh_single_x4_s8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_sat_double_mulh_single_x4_s8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_sat_double_mulh_single_x4_s8: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -75,8 +75,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_sat_double_mulh_single_x4_s16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_sat_double_mulh_single_x4_s16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_sat_double_mulh_single_x4_s16: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -94,8 +93,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_sat_double_mulh_single_x4_s32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_sat_double_mulh_single_x4_s32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_sat_double_mulh_single_x4_s32: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -113,8 +111,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_sat_double_mulh_single_x4_s64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_sat_double_mulh_single_x4_s64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_sat_double_mulh_single_x4_s64: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -196,104 +193,100 @@ define { , } @multi_vec_sat_double_mulh_mul ; SQDMULH (x4, Multi) -define { , , , } -@multi_vec_sat_double_mulh_multi_x4_s8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_sat_double_mulh_multi_x4_s8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_sat_double_mulh_multi_x4_s8: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.b -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1b { z31.b }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: sqdmulh { z24.b - z27.b }, { z24.b - z27.b }, { z28.b - z31.b } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.sqdmulh.vgx4.nxv16i8( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_sat_double_mulh_multi_x4_s16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_sat_double_mulh_multi_x4_s16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_sat_double_mulh_multi_x4_s16: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.h -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1h { z31.h }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: sqdmulh { z24.h - z27.h }, { z24.h - z27.h }, { z28.h - z31.h } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.sqdmulh.vgx4.nxv8i16( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_sat_double_mulh_multi_x4_s32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_sat_double_mulh_multi_x4_s32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_sat_double_mulh_multi_x4_s32: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1w { z31.s }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: sqdmulh { z24.s - z27.s }, { z24.s - z27.s }, { z28.s - z31.s } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.sqdmulh.vgx4.nxv4i32( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_sat_double_mulh_multi_x4_s64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_sat_double_mulh_multi_x4_s64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_sat_double_mulh_multi_x4_s64: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1d { z31.d }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: sqdmulh { z24.d - z27.d }, { z24.d - z27.d }, { z28.d - z31.d } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.sqdmulh.vgx4.nxv2i64( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) diff --git a/llvm/test/CodeGen/AArch64/split-vector-insert.ll b/llvm/test/CodeGen/AArch64/split-vector-insert.ll index a507296338f9..b8ab9a00c698 100644 --- a/llvm/test/CodeGen/AArch64/split-vector-insert.ll +++ b/llvm/test/CodeGen/AArch64/split-vector-insert.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 ; RUN: llc < %s -debug-only=legalize-types 2>&1 | FileCheck %s --check-prefix=CHECK-LEGALIZATION ; RUN: llc < %s | FileCheck %s ; REQUIRES: asserts @@ -9,54 +10,94 @@ declare @llvm.vector.insert.nxv2i64.v8i64(, declare @llvm.vector.insert.nxv2f64.v8f64(, <8 x double>, i64) define @test_nxv2i64_v8i64( %a, <8 x i64> %b) #0 { -; CHECK-LEGALIZATION: Legally typed node: [[T1:t[0-9]+]]: nxv2i64 = insert_subvector {{t[0-9]+}}, {{t[0-9]+}}, Constant:i64<0> -; CHECK-LEGALIZATION: Legally typed node: [[T2:t[0-9]+]]: nxv2i64 = insert_subvector [[T1]], {{t[0-9]+}}, Constant:i64<2> -; CHECK-LEGALIZATION: Legally typed node: [[T3:t[0-9]+]]: nxv2i64 = insert_subvector [[T2]], {{t[0-9]+}}, Constant:i64<4> -; CHECK-LEGALIZATION: Legally typed node: [[T4:t[0-9]+]]: nxv2i64 = insert_subvector [[T3]], {{t[0-9]+}}, Constant:i64<6> - +; CHECK-LEGALIZATION-LABEL: test_nxv2i64_v8i64: +; CHECK-LEGALIZATION: // %bb.0: +; CHECK-LEGALIZATION-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-LEGALIZATION-NEXT: .cfi_def_cfa_offset 16 +; CHECK-LEGALIZATION-NEXT: .cfi_offset w29, -16 +; CHECK-LEGALIZATION-NEXT: addvl sp, sp, #-3 +; CHECK-LEGALIZATION-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG +; CHECK-LEGALIZATION-NEXT: cntd x8 +; CHECK-LEGALIZATION-NEXT: ptrue p0.d, vl2 +; CHECK-LEGALIZATION-NEXT: mov w9, #2 // =0x2 +; CHECK-LEGALIZATION-NEXT: sub x8, x8, #2 +; CHECK-LEGALIZATION-NEXT: // kill: def $q1 killed $q1 def $z1 +; CHECK-LEGALIZATION-NEXT: mov x10, sp +; CHECK-LEGALIZATION-NEXT: cmp x8, #2 +; CHECK-LEGALIZATION-NEXT: mov z0.d, p0/m, z1.d +; CHECK-LEGALIZATION-NEXT: ptrue p0.d +; CHECK-LEGALIZATION-NEXT: csel x9, x8, x9, lo +; CHECK-LEGALIZATION-NEXT: cmp x8, #4 +; CHECK-LEGALIZATION-NEXT: lsl x9, x9, #3 +; CHECK-LEGALIZATION-NEXT: st1d { z0.d }, p0, [sp] +; CHECK-LEGALIZATION-NEXT: str q2, [x10, x9] +; CHECK-LEGALIZATION-NEXT: mov w9, #4 // =0x4 +; CHECK-LEGALIZATION-NEXT: addvl x10, sp, #1 +; CHECK-LEGALIZATION-NEXT: ld1d { z0.d }, p0/z, [sp] +; CHECK-LEGALIZATION-NEXT: csel x9, x8, x9, lo +; CHECK-LEGALIZATION-NEXT: cmp x8, #6 +; CHECK-LEGALIZATION-NEXT: lsl x9, x9, #3 +; CHECK-LEGALIZATION-NEXT: st1d { z0.d }, p0, [sp, #1, mul vl] +; CHECK-LEGALIZATION-NEXT: str q3, [x10, x9] +; CHECK-LEGALIZATION-NEXT: mov w9, #6 // =0x6 +; CHECK-LEGALIZATION-NEXT: ld1d { z0.d }, p0/z, [sp, #1, mul vl] +; CHECK-LEGALIZATION-NEXT: csel x8, x8, x9, lo +; CHECK-LEGALIZATION-NEXT: addvl x9, sp, #2 +; CHECK-LEGALIZATION-NEXT: lsl x8, x8, #3 +; CHECK-LEGALIZATION-NEXT: st1d { z0.d }, p0, [sp, #2, mul vl] +; CHECK-LEGALIZATION-NEXT: str q4, [x9, x8] +; CHECK-LEGALIZATION-NEXT: ld1d { z0.d }, p0/z, [sp, #2, mul vl] +; CHECK-LEGALIZATION-NEXT: addvl sp, sp, #3 +; CHECK-LEGALIZATION-NEXT: .cfi_def_cfa wsp, 16 +; CHECK-LEGALIZATION-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-LEGALIZATION-NEXT: .cfi_def_cfa_offset 0 +; CHECK-LEGALIZATION-NEXT: .cfi_restore w29 +; CHECK-LEGALIZATION-NEXT: ret +; ; CHECK-LABEL: test_nxv2i64_v8i64: ; CHECK: // %bb.0: -; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill -; CHECK-NEXT: .cfi_def_cfa_offset 16 -; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: addvl sp, sp, #-3 -; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG -; CHECK-NEXT: ptrue p1.d, vl2 -; CHECK-NEXT: cntd x8 -; CHECK-NEXT: mov w9, #2 // =0x2 -; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: sub x8, x8, #2 -; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 -; CHECK-NEXT: mov x10, sp -; CHECK-NEXT: cmp x8, #2 -; CHECK-NEXT: csel x9, x8, x9, lo -; CHECK-NEXT: cmp x8, #4 -; CHECK-NEXT: lsl x9, x9, #3 -; CHECK-NEXT: mov z0.d, p1/m, z1.d -; CHECK-NEXT: st1d { z0.d }, p0, [sp] -; CHECK-NEXT: str q2, [x10, x9] -; CHECK-NEXT: mov w9, #4 // =0x4 -; CHECK-NEXT: addvl x10, sp, #1 -; CHECK-NEXT: ld1d { z0.d }, p0/z, [sp] -; CHECK-NEXT: csel x9, x8, x9, lo -; CHECK-NEXT: cmp x8, #6 -; CHECK-NEXT: lsl x9, x9, #3 -; CHECK-NEXT: st1d { z0.d }, p0, [sp, #1, mul vl] -; CHECK-NEXT: str q3, [x10, x9] -; CHECK-NEXT: mov w9, #6 // =0x6 -; CHECK-NEXT: ld1d { z0.d }, p0/z, [sp, #1, mul vl] -; CHECK-NEXT: csel x8, x8, x9, lo -; CHECK-NEXT: addvl x9, sp, #2 -; CHECK-NEXT: lsl x8, x8, #3 -; CHECK-NEXT: st1d { z0.d }, p0, [sp, #2, mul vl] -; CHECK-NEXT: str q4, [x9, x8] -; CHECK-NEXT: ld1d { z0.d }, p0/z, [sp, #2, mul vl] -; CHECK-NEXT: addvl sp, sp, #3 -; CHECK-NEXT: .cfi_def_cfa wsp, 16 -; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload -; CHECK-NEXT: .cfi_def_cfa_offset 0 -; CHECK-NEXT: .cfi_restore w29 -; CHECK-NEXT: ret +; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: .cfi_offset w29, -16 +; CHECK-NEXT: addvl sp, sp, #-3 +; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG +; CHECK-NEXT: cntd x8 +; CHECK-NEXT: ptrue p0.d, vl2 +; CHECK-NEXT: mov w9, #2 // =0x2 +; CHECK-NEXT: sub x8, x8, #2 +; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 +; CHECK-NEXT: mov x10, sp +; CHECK-NEXT: cmp x8, #2 +; CHECK-NEXT: mov z0.d, p0/m, z1.d +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: csel x9, x8, x9, lo +; CHECK-NEXT: cmp x8, #4 +; CHECK-NEXT: lsl x9, x9, #3 +; CHECK-NEXT: st1d { z0.d }, p0, [sp] +; CHECK-NEXT: str q2, [x10, x9] +; CHECK-NEXT: mov w9, #4 // =0x4 +; CHECK-NEXT: addvl x10, sp, #1 +; CHECK-NEXT: ld1d { z0.d }, p0/z, [sp] +; CHECK-NEXT: csel x9, x8, x9, lo +; CHECK-NEXT: cmp x8, #6 +; CHECK-NEXT: lsl x9, x9, #3 +; CHECK-NEXT: st1d { z0.d }, p0, [sp, #1, mul vl] +; CHECK-NEXT: str q3, [x10, x9] +; CHECK-NEXT: mov w9, #6 // =0x6 +; CHECK-NEXT: ld1d { z0.d }, p0/z, [sp, #1, mul vl] +; CHECK-NEXT: csel x8, x8, x9, lo +; CHECK-NEXT: addvl x9, sp, #2 +; CHECK-NEXT: lsl x8, x8, #3 +; CHECK-NEXT: st1d { z0.d }, p0, [sp, #2, mul vl] +; CHECK-NEXT: str q4, [x9, x8] +; CHECK-NEXT: ld1d { z0.d }, p0/z, [sp, #2, mul vl] +; CHECK-NEXT: addvl sp, sp, #3 +; CHECK-NEXT: .cfi_def_cfa wsp, 16 +; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w29 +; CHECK-NEXT: ret + @@ -66,54 +107,94 @@ define @test_nxv2i64_v8i64( %a, <8 x i64> % } define @test_nxv2f64_v8f64( %a, <8 x double> %b) #0 { -; CHECK-LEGALIZATION: Legally typed node: [[T1:t[0-9]+]]: nxv2f64 = insert_subvector {{t[0-9]+}}, {{t[0-9]+}}, Constant:i64<0> -; CHECK-LEGALIZATION: Legally typed node: [[T2:t[0-9]+]]: nxv2f64 = insert_subvector [[T1]], {{t[0-9]+}}, Constant:i64<2> -; CHECK-LEGALIZATION: Legally typed node: [[T3:t[0-9]+]]: nxv2f64 = insert_subvector [[T2]], {{t[0-9]+}}, Constant:i64<4> -; CHECK-LEGALIZATION: Legally typed node: [[T4:t[0-9]+]]: nxv2f64 = insert_subvector [[T3]], {{t[0-9]+}}, Constant:i64<6> - +; CHECK-LEGALIZATION-LABEL: test_nxv2f64_v8f64: +; CHECK-LEGALIZATION: // %bb.0: +; CHECK-LEGALIZATION-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-LEGALIZATION-NEXT: .cfi_def_cfa_offset 16 +; CHECK-LEGALIZATION-NEXT: .cfi_offset w29, -16 +; CHECK-LEGALIZATION-NEXT: addvl sp, sp, #-3 +; CHECK-LEGALIZATION-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG +; CHECK-LEGALIZATION-NEXT: cntd x8 +; CHECK-LEGALIZATION-NEXT: ptrue p0.d, vl2 +; CHECK-LEGALIZATION-NEXT: mov w9, #2 // =0x2 +; CHECK-LEGALIZATION-NEXT: sub x8, x8, #2 +; CHECK-LEGALIZATION-NEXT: // kill: def $q1 killed $q1 def $z1 +; CHECK-LEGALIZATION-NEXT: mov x10, sp +; CHECK-LEGALIZATION-NEXT: cmp x8, #2 +; CHECK-LEGALIZATION-NEXT: mov z0.d, p0/m, z1.d +; CHECK-LEGALIZATION-NEXT: ptrue p0.d +; CHECK-LEGALIZATION-NEXT: csel x9, x8, x9, lo +; CHECK-LEGALIZATION-NEXT: cmp x8, #4 +; CHECK-LEGALIZATION-NEXT: lsl x9, x9, #3 +; CHECK-LEGALIZATION-NEXT: st1d { z0.d }, p0, [sp] +; CHECK-LEGALIZATION-NEXT: str q2, [x10, x9] +; CHECK-LEGALIZATION-NEXT: mov w9, #4 // =0x4 +; CHECK-LEGALIZATION-NEXT: addvl x10, sp, #1 +; CHECK-LEGALIZATION-NEXT: ld1d { z0.d }, p0/z, [sp] +; CHECK-LEGALIZATION-NEXT: csel x9, x8, x9, lo +; CHECK-LEGALIZATION-NEXT: cmp x8, #6 +; CHECK-LEGALIZATION-NEXT: lsl x9, x9, #3 +; CHECK-LEGALIZATION-NEXT: st1d { z0.d }, p0, [sp, #1, mul vl] +; CHECK-LEGALIZATION-NEXT: str q3, [x10, x9] +; CHECK-LEGALIZATION-NEXT: mov w9, #6 // =0x6 +; CHECK-LEGALIZATION-NEXT: ld1d { z0.d }, p0/z, [sp, #1, mul vl] +; CHECK-LEGALIZATION-NEXT: csel x8, x8, x9, lo +; CHECK-LEGALIZATION-NEXT: addvl x9, sp, #2 +; CHECK-LEGALIZATION-NEXT: lsl x8, x8, #3 +; CHECK-LEGALIZATION-NEXT: st1d { z0.d }, p0, [sp, #2, mul vl] +; CHECK-LEGALIZATION-NEXT: str q4, [x9, x8] +; CHECK-LEGALIZATION-NEXT: ld1d { z0.d }, p0/z, [sp, #2, mul vl] +; CHECK-LEGALIZATION-NEXT: addvl sp, sp, #3 +; CHECK-LEGALIZATION-NEXT: .cfi_def_cfa wsp, 16 +; CHECK-LEGALIZATION-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-LEGALIZATION-NEXT: .cfi_def_cfa_offset 0 +; CHECK-LEGALIZATION-NEXT: .cfi_restore w29 +; CHECK-LEGALIZATION-NEXT: ret +; ; CHECK-LABEL: test_nxv2f64_v8f64: ; CHECK: // %bb.0: -; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill -; CHECK-NEXT: .cfi_def_cfa_offset 16 -; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: addvl sp, sp, #-3 -; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG -; CHECK-NEXT: ptrue p1.d, vl2 -; CHECK-NEXT: cntd x8 -; CHECK-NEXT: mov w9, #2 // =0x2 -; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: sub x8, x8, #2 -; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 -; CHECK-NEXT: mov x10, sp -; CHECK-NEXT: cmp x8, #2 -; CHECK-NEXT: csel x9, x8, x9, lo -; CHECK-NEXT: cmp x8, #4 -; CHECK-NEXT: lsl x9, x9, #3 -; CHECK-NEXT: mov z0.d, p1/m, z1.d -; CHECK-NEXT: st1d { z0.d }, p0, [sp] -; CHECK-NEXT: str q2, [x10, x9] -; CHECK-NEXT: mov w9, #4 // =0x4 -; CHECK-NEXT: addvl x10, sp, #1 -; CHECK-NEXT: ld1d { z0.d }, p0/z, [sp] -; CHECK-NEXT: csel x9, x8, x9, lo -; CHECK-NEXT: cmp x8, #6 -; CHECK-NEXT: lsl x9, x9, #3 -; CHECK-NEXT: st1d { z0.d }, p0, [sp, #1, mul vl] -; CHECK-NEXT: str q3, [x10, x9] -; CHECK-NEXT: mov w9, #6 // =0x6 -; CHECK-NEXT: ld1d { z0.d }, p0/z, [sp, #1, mul vl] -; CHECK-NEXT: csel x8, x8, x9, lo -; CHECK-NEXT: addvl x9, sp, #2 -; CHECK-NEXT: lsl x8, x8, #3 -; CHECK-NEXT: st1d { z0.d }, p0, [sp, #2, mul vl] -; CHECK-NEXT: str q4, [x9, x8] -; CHECK-NEXT: ld1d { z0.d }, p0/z, [sp, #2, mul vl] -; CHECK-NEXT: addvl sp, sp, #3 -; CHECK-NEXT: .cfi_def_cfa wsp, 16 -; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload -; CHECK-NEXT: .cfi_def_cfa_offset 0 -; CHECK-NEXT: .cfi_restore w29 -; CHECK-NEXT: ret +; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: .cfi_offset w29, -16 +; CHECK-NEXT: addvl sp, sp, #-3 +; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG +; CHECK-NEXT: cntd x8 +; CHECK-NEXT: ptrue p0.d, vl2 +; CHECK-NEXT: mov w9, #2 // =0x2 +; CHECK-NEXT: sub x8, x8, #2 +; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 +; CHECK-NEXT: mov x10, sp +; CHECK-NEXT: cmp x8, #2 +; CHECK-NEXT: mov z0.d, p0/m, z1.d +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: csel x9, x8, x9, lo +; CHECK-NEXT: cmp x8, #4 +; CHECK-NEXT: lsl x9, x9, #3 +; CHECK-NEXT: st1d { z0.d }, p0, [sp] +; CHECK-NEXT: str q2, [x10, x9] +; CHECK-NEXT: mov w9, #4 // =0x4 +; CHECK-NEXT: addvl x10, sp, #1 +; CHECK-NEXT: ld1d { z0.d }, p0/z, [sp] +; CHECK-NEXT: csel x9, x8, x9, lo +; CHECK-NEXT: cmp x8, #6 +; CHECK-NEXT: lsl x9, x9, #3 +; CHECK-NEXT: st1d { z0.d }, p0, [sp, #1, mul vl] +; CHECK-NEXT: str q3, [x10, x9] +; CHECK-NEXT: mov w9, #6 // =0x6 +; CHECK-NEXT: ld1d { z0.d }, p0/z, [sp, #1, mul vl] +; CHECK-NEXT: csel x8, x8, x9, lo +; CHECK-NEXT: addvl x9, sp, #2 +; CHECK-NEXT: lsl x8, x8, #3 +; CHECK-NEXT: st1d { z0.d }, p0, [sp, #2, mul vl] +; CHECK-NEXT: str q4, [x9, x8] +; CHECK-NEXT: ld1d { z0.d }, p0/z, [sp, #2, mul vl] +; CHECK-NEXT: addvl sp, sp, #3 +; CHECK-NEXT: .cfi_def_cfa wsp, 16 +; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w29 +; CHECK-NEXT: ret + diff --git a/llvm/test/CodeGen/AArch64/srem-seteq-vec-splat.ll b/llvm/test/CodeGen/AArch64/srem-seteq-vec-splat.ll index 1d9cb88260b6..c0c0ae5c9d1f 100644 --- a/llvm/test/CodeGen/AArch64/srem-seteq-vec-splat.ll +++ b/llvm/test/CodeGen/AArch64/srem-seteq-vec-splat.ll @@ -111,6 +111,7 @@ define <4 x i32> @test_srem_odd_undef1(<4 x i32> %X) nounwind { ; CHECK-LABEL: test_srem_odd_undef1: ; CHECK: // %bb.0: ; CHECK-NEXT: mov w8, #34079 // =0x851f +; CHECK-NEXT: movi v3.4s, #25 ; CHECK-NEXT: movk w8, #20971, lsl #16 ; CHECK-NEXT: dup v1.4s, w8 ; CHECK-NEXT: smull2 v2.2d, v0.4s, v1.4s @@ -118,9 +119,8 @@ define <4 x i32> @test_srem_odd_undef1(<4 x i32> %X) nounwind { ; CHECK-NEXT: uzp2 v1.4s, v1.4s, v2.4s ; CHECK-NEXT: sshr v2.4s, v1.4s, #3 ; CHECK-NEXT: usra v2.4s, v1.4s, #31 -; CHECK-NEXT: movi v1.4s, #25 -; CHECK-NEXT: mls v0.4s, v2.4s, v1.4s ; CHECK-NEXT: movi v1.4s, #1 +; CHECK-NEXT: mls v0.4s, v2.4s, v3.4s ; CHECK-NEXT: cmeq v0.4s, v0.4s, #0 ; CHECK-NEXT: and v0.16b, v0.16b, v1.16b ; CHECK-NEXT: ret @@ -134,6 +134,7 @@ define <4 x i32> @test_srem_even_undef1(<4 x i32> %X) nounwind { ; CHECK-LABEL: test_srem_even_undef1: ; CHECK: // %bb.0: ; CHECK-NEXT: mov w8, #34079 // =0x851f +; CHECK-NEXT: movi v3.4s, #100 ; CHECK-NEXT: movk w8, #20971, lsl #16 ; CHECK-NEXT: dup v1.4s, w8 ; CHECK-NEXT: smull2 v2.2d, v0.4s, v1.4s @@ -141,9 +142,8 @@ define <4 x i32> @test_srem_even_undef1(<4 x i32> %X) nounwind { ; CHECK-NEXT: uzp2 v1.4s, v1.4s, v2.4s ; CHECK-NEXT: sshr v2.4s, v1.4s, #5 ; CHECK-NEXT: usra v2.4s, v1.4s, #31 -; CHECK-NEXT: movi v1.4s, #100 -; CHECK-NEXT: mls v0.4s, v2.4s, v1.4s ; CHECK-NEXT: movi v1.4s, #1 +; CHECK-NEXT: mls v0.4s, v2.4s, v3.4s ; CHECK-NEXT: cmeq v0.4s, v0.4s, #0 ; CHECK-NEXT: and v0.16b, v0.16b, v1.16b ; CHECK-NEXT: ret @@ -201,11 +201,11 @@ define <4 x i32> @test_srem_pow2(<4 x i32> %X) nounwind { define <4 x i32> @test_srem_int_min(<4 x i32> %X) nounwind { ; CHECK-LABEL: test_srem_int_min: ; CHECK: // %bb.0: -; CHECK-NEXT: cmlt v1.4s, v0.4s, #0 -; CHECK-NEXT: mov v2.16b, v0.16b -; CHECK-NEXT: usra v2.4s, v1.4s, #1 +; CHECK-NEXT: cmlt v2.4s, v0.4s, #0 +; CHECK-NEXT: mov v3.16b, v0.16b ; CHECK-NEXT: movi v1.4s, #128, lsl #24 -; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: usra v3.4s, v2.4s, #1 +; CHECK-NEXT: and v1.16b, v3.16b, v1.16b ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s ; CHECK-NEXT: movi v1.4s, #1 ; CHECK-NEXT: cmeq v0.4s, v0.4s, #0 diff --git a/llvm/test/CodeGen/AArch64/srem-vector-lkk.ll b/llvm/test/CodeGen/AArch64/srem-vector-lkk.ll index 0598af7c9806..a74f0c86fe18 100644 --- a/llvm/test/CodeGen/AArch64/srem-vector-lkk.ll +++ b/llvm/test/CodeGen/AArch64/srem-vector-lkk.ll @@ -245,6 +245,7 @@ define <4 x i32> @fold_srem_v4i32(<4 x i32> %x) { ; CHECK-LABEL: fold_srem_v4i32: ; CHECK: // %bb.0: ; CHECK-NEXT: mov w8, #26215 // =0x6667 +; CHECK-NEXT: movi v3.4s, #10 ; CHECK-NEXT: movk w8, #26214, lsl #16 ; CHECK-NEXT: dup v1.4s, w8 ; CHECK-NEXT: smull2 v2.2d, v0.4s, v1.4s @@ -252,8 +253,7 @@ define <4 x i32> @fold_srem_v4i32(<4 x i32> %x) { ; CHECK-NEXT: uzp2 v1.4s, v1.4s, v2.4s ; CHECK-NEXT: sshr v2.4s, v1.4s, #2 ; CHECK-NEXT: usra v2.4s, v1.4s, #31 -; CHECK-NEXT: movi v1.4s, #10 -; CHECK-NEXT: mls v0.4s, v2.4s, v1.4s +; CHECK-NEXT: mls v0.4s, v2.4s, v3.4s ; CHECK-NEXT: ret %1 = srem <4 x i32> %x, ret <4 x i32> %1 diff --git a/llvm/test/CodeGen/AArch64/sve-abd.ll b/llvm/test/CodeGen/AArch64/sve-abd.ll index 95aec0a49261..7b492229e3d2 100644 --- a/llvm/test/CodeGen/AArch64/sve-abd.ll +++ b/llvm/test/CodeGen/AArch64/sve-abd.ll @@ -24,10 +24,10 @@ define @sabd_b( %a, %b) define @sabd_b_promoted_ops( %a, %b) #0 { ; CHECK-LABEL: sabd_b_promoted_ops: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p2.b ; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z1.b, p1/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: sabd z0.b, p2/m, z0.b, z1.b +; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: sabd z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: ret %a.sext = sext %a to %b.sext = sext %b to @@ -144,10 +144,10 @@ define @uabd_b( %a, %b) define @uabd_b_promoted_ops( %a, %b) #0 { ; CHECK-LABEL: uabd_b_promoted_ops: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p2.b ; CHECK-NEXT: mov z0.b, p0/z, #1 // =0x1 ; CHECK-NEXT: mov z1.b, p1/z, #1 // =0x1 -; CHECK-NEXT: uabd z0.b, p2/m, z0.b, z1.b +; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: uabd z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: ret %a.zext = zext %a to %b.zext = zext %b to @@ -173,9 +173,9 @@ define @uabd_h( %a, %b) define @uabd_h_promoted_ops( %a, %b) #0 { ; CHECK-LABEL: uabd_h_promoted_ops: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: and z0.h, z0.h, #0xff ; CHECK-NEXT: and z1.h, z1.h, #0xff +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: uabd z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret %a.zext = zext %a to @@ -202,9 +202,9 @@ define @uabd_s( %a, %b) define @uabd_s_promoted_ops( %a, %b) #0 { ; CHECK-LABEL: uabd_s_promoted_ops: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and z0.s, z0.s, #0xffff ; CHECK-NEXT: and z1.s, z1.s, #0xffff +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uabd z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %a.zext = zext %a to @@ -231,9 +231,9 @@ define @uabd_d( %a, %b) define @uabd_d_promoted_ops( %a, %b) #0 { ; CHECK-LABEL: uabd_d_promoted_ops: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and z0.d, z0.d, #0xffffffff ; CHECK-NEXT: and z1.d, z1.d, #0xffffffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: uabd z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret %a.zext = zext %a to @@ -248,8 +248,8 @@ define @uabd_d_promoted_ops( %a, @uabd_non_matching_extension( %a, %b) #0 { ; CHECK-LABEL: uabd_non_matching_extension: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and z1.s, z1.s, #0xff +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uabd z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %a.zext = zext %a to @@ -265,9 +265,9 @@ define @uabd_non_matching_extension( %a, @uabd_non_matching_promoted_ops( %a, %b) #0 { ; CHECK-LABEL: uabd_non_matching_promoted_ops: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and z0.s, z0.s, #0xff ; CHECK-NEXT: and z1.s, z1.s, #0xffff +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uabd z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %a.zext = zext %a to diff --git a/llvm/test/CodeGen/AArch64/sve-bitcast.ll b/llvm/test/CodeGen/AArch64/sve-bitcast.ll index 7dd568fc837a..95f43ba51263 100644 --- a/llvm/test/CodeGen/AArch64/sve-bitcast.ll +++ b/llvm/test/CodeGen/AArch64/sve-bitcast.ll @@ -1698,8 +1698,8 @@ define @bitcast_nxv8i8_to_nxv1i64( %v) #0 { ; CHECK_BE: // %bb.0: ; CHECK_BE-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK_BE-NEXT: addvl sp, sp, #-1 -; CHECK_BE-NEXT: ptrue p0.b ; CHECK_BE-NEXT: uzp1 z0.b, z0.b, z0.b +; CHECK_BE-NEXT: ptrue p0.b ; CHECK_BE-NEXT: ptrue p1.d ; CHECK_BE-NEXT: st1b { z0.b }, p0, [sp] ; CHECK_BE-NEXT: ld1d { z0.d }, p1/z, [sp] @@ -1720,8 +1720,8 @@ define @bitcast_nxv4i16_to_nxv1i64( %v) #0 ; CHECK_BE: // %bb.0: ; CHECK_BE-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK_BE-NEXT: addvl sp, sp, #-1 -; CHECK_BE-NEXT: ptrue p0.h ; CHECK_BE-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK_BE-NEXT: ptrue p0.h ; CHECK_BE-NEXT: ptrue p1.d ; CHECK_BE-NEXT: st1h { z0.h }, p0, [sp] ; CHECK_BE-NEXT: ld1d { z0.d }, p1/z, [sp] @@ -1742,8 +1742,8 @@ define @bitcast_nxv2i32_to_nxv1i64( %v) #0 ; CHECK_BE: // %bb.0: ; CHECK_BE-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK_BE-NEXT: addvl sp, sp, #-1 -; CHECK_BE-NEXT: ptrue p0.s ; CHECK_BE-NEXT: uzp1 z0.s, z0.s, z0.s +; CHECK_BE-NEXT: ptrue p0.s ; CHECK_BE-NEXT: ptrue p1.d ; CHECK_BE-NEXT: st1w { z0.s }, p0, [sp] ; CHECK_BE-NEXT: ld1d { z0.d }, p1/z, [sp] @@ -2218,8 +2218,8 @@ define @bitcast_nxv8i8_to_nxv1f64( %v) #0 ; CHECK_BE: // %bb.0: ; CHECK_BE-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK_BE-NEXT: addvl sp, sp, #-1 -; CHECK_BE-NEXT: ptrue p0.b ; CHECK_BE-NEXT: uzp1 z0.b, z0.b, z0.b +; CHECK_BE-NEXT: ptrue p0.b ; CHECK_BE-NEXT: ptrue p1.d ; CHECK_BE-NEXT: st1b { z0.b }, p0, [sp] ; CHECK_BE-NEXT: ld1d { z0.d }, p1/z, [sp] @@ -2240,8 +2240,8 @@ define @bitcast_nxv4i16_to_nxv1f64( %v) ; CHECK_BE: // %bb.0: ; CHECK_BE-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK_BE-NEXT: addvl sp, sp, #-1 -; CHECK_BE-NEXT: ptrue p0.h ; CHECK_BE-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK_BE-NEXT: ptrue p0.h ; CHECK_BE-NEXT: ptrue p1.d ; CHECK_BE-NEXT: st1h { z0.h }, p0, [sp] ; CHECK_BE-NEXT: ld1d { z0.d }, p1/z, [sp] @@ -2262,8 +2262,8 @@ define @bitcast_nxv2i32_to_nxv1f64( %v) ; CHECK_BE: // %bb.0: ; CHECK_BE-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK_BE-NEXT: addvl sp, sp, #-1 -; CHECK_BE-NEXT: ptrue p0.s ; CHECK_BE-NEXT: uzp1 z0.s, z0.s, z0.s +; CHECK_BE-NEXT: ptrue p0.s ; CHECK_BE-NEXT: ptrue p1.d ; CHECK_BE-NEXT: st1w { z0.s }, p0, [sp] ; CHECK_BE-NEXT: ld1d { z0.d }, p1/z, [sp] @@ -2827,11 +2827,11 @@ define @bitcast_nxv2f16_to_nxv1i32( %v) #0 ; CHECK_BE-NEXT: addvl sp, sp, #-2 ; CHECK_BE-NEXT: ptrue p0.d ; CHECK_BE-NEXT: ptrue p1.h -; CHECK_BE-NEXT: ptrue p2.s ; CHECK_BE-NEXT: st1h { z0.d }, p0, [sp] +; CHECK_BE-NEXT: ptrue p0.s ; CHECK_BE-NEXT: ld1h { z0.h }, p1/z, [sp] ; CHECK_BE-NEXT: st1h { z0.h }, p1, [sp, #1, mul vl] -; CHECK_BE-NEXT: ld1w { z0.s }, p2/z, [sp, #1, mul vl] +; CHECK_BE-NEXT: ld1w { z0.s }, p0/z, [sp, #1, mul vl] ; CHECK_BE-NEXT: addvl sp, sp, #2 ; CHECK_BE-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload ; CHECK_BE-NEXT: ret @@ -2860,11 +2860,11 @@ define @bitcast_nxv2bf16_to_nxv1i32( %v) ; CHECK_BE-NEXT: addvl sp, sp, #-2 ; CHECK_BE-NEXT: ptrue p0.d ; CHECK_BE-NEXT: ptrue p1.h -; CHECK_BE-NEXT: ptrue p2.s ; CHECK_BE-NEXT: st1h { z0.d }, p0, [sp] +; CHECK_BE-NEXT: ptrue p0.s ; CHECK_BE-NEXT: ld1h { z0.h }, p1/z, [sp] ; CHECK_BE-NEXT: st1h { z0.h }, p1, [sp, #1, mul vl] -; CHECK_BE-NEXT: ld1w { z0.s }, p2/z, [sp, #1, mul vl] +; CHECK_BE-NEXT: ld1w { z0.s }, p0/z, [sp, #1, mul vl] ; CHECK_BE-NEXT: addvl sp, sp, #2 ; CHECK_BE-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload ; CHECK_BE-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sve-calling-convention-mixed.ll b/llvm/test/CodeGen/AArch64/sve-calling-convention-mixed.ll index 56b023086ea2..3b7b03e6ef61 100644 --- a/llvm/test/CodeGen/AArch64/sve-calling-convention-mixed.ll +++ b/llvm/test/CodeGen/AArch64/sve-calling-convention-mixed.ll @@ -64,12 +64,12 @@ define float @foo2(ptr %x0, ptr %x1) nounwind { ; CHECK-NEXT: add x9, sp, #16 ; CHECK-NEXT: mov w2, #2 // =0x2 ; CHECK-NEXT: mov w3, #3 // =0x3 +; CHECK-NEXT: ld4d { z1.d - z4.d }, p0/z, [x0] +; CHECK-NEXT: mov w0, wzr ; CHECK-NEXT: mov w4, #4 // =0x4 ; CHECK-NEXT: mov w5, #5 // =0x5 ; CHECK-NEXT: mov w6, #6 // =0x6 ; CHECK-NEXT: mov w7, #7 // =0x7 -; CHECK-NEXT: ld4d { z1.d - z4.d }, p0/z, [x0] -; CHECK-NEXT: mov w0, wzr ; CHECK-NEXT: ld4d { z16.d - z19.d }, p0/z, [x1] ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w1, #1 // =0x1 @@ -182,8 +182,8 @@ entry: define double @foo5(i32 %i0, i32 %i1, i32 %i2, i32 %i3, i32 %i4, i32 %i5, ptr %ptr1, ptr %ptr2, double %x0, %x1, %x2) nounwind { ; CHECK-LABEL: foo5: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldr x8, [sp] +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ld1d { z5.d }, p0/z, [x8, #1, mul vl] ; CHECK-NEXT: ld1d { z6.d }, p0/z, [x8] ; CHECK-NEXT: ld1d { z7.d }, p0/z, [x8, #3, mul vl] @@ -229,10 +229,10 @@ entry: define void @aavpcs1(i32 %s0, i32 %s1, i32 %s2, i32 %s3, i32 %s4, i32 %s5, i32 %s6, %s7, %s8, %s9, %s10, %s11, %s12, %s13, %s14, %s15, %s16, ptr %ptr) nounwind { ; CHECK-LABEL: aavpcs1: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldp x8, x9, [sp] -; CHECK-NEXT: ld1w { z3.s }, p0/z, [x8] +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ld1w { z24.s }, p0/z, [x7] +; CHECK-NEXT: ld1w { z3.s }, p0/z, [x8] ; CHECK-NEXT: st1w { z0.s }, p0, [x9] ; CHECK-NEXT: st1w { z1.s }, p0, [x9] ; CHECK-NEXT: st1w { z2.s }, p0, [x9] @@ -261,12 +261,12 @@ entry: define void @aavpcs2(float %s0, float %s1, float %s2, float %s3, float %s4, float %s5, float %s6, %s7, %s8, %s9, %s10, %s11, %s12, %s13, %s14, %s15, %s16,ptr %ptr) nounwind { ; CHECK-LABEL: aavpcs2: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldp x8, x9, [sp] -; CHECK-NEXT: ld1w { z0.s }, p0/z, [x8] +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ld1w { z1.s }, p0/z, [x7] ; CHECK-NEXT: ld1w { z2.s }, p0/z, [x0] ; CHECK-NEXT: ld1w { z3.s }, p0/z, [x6] +; CHECK-NEXT: ld1w { z0.s }, p0/z, [x8] ; CHECK-NEXT: ld1w { z4.s }, p0/z, [x5] ; CHECK-NEXT: ld1w { z5.s }, p0/z, [x1] ; CHECK-NEXT: ld1w { z6.s }, p0/z, [x4] @@ -299,11 +299,10 @@ entry: define void @aavpcs3(float %s0, float %s1, float %s2, float %s3, float %s4, float %s5, float %s6, float %s7, %s8, %s9, %s10, %s11, %s12, %s13, %s14, %s15, %s16, %s17, %p0, ptr %ptr) nounwind { ; CHECK-LABEL: aavpcs3: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldr x8, [sp] -; CHECK-NEXT: ldr x9, [sp, #16] -; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0] -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x8] +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: ld1w { z0.s }, p0/z, [x8] +; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] ; CHECK-NEXT: ld1w { z2.s }, p0/z, [x7] ; CHECK-NEXT: ld1w { z3.s }, p0/z, [x1] ; CHECK-NEXT: ld1w { z4.s }, p0/z, [x6] @@ -311,15 +310,16 @@ define void @aavpcs3(float %s0, float %s1, float %s2, float %s3, float %s4, floa ; CHECK-NEXT: ld1w { z6.s }, p0/z, [x2] ; CHECK-NEXT: ld1w { z7.s }, p0/z, [x4] ; CHECK-NEXT: ld1w { z24.s }, p0/z, [x3] -; CHECK-NEXT: st1w { z0.s }, p0, [x9] -; CHECK-NEXT: st1w { z3.s }, p0, [x9] -; CHECK-NEXT: st1w { z6.s }, p0, [x9] -; CHECK-NEXT: st1w { z24.s }, p0, [x9] -; CHECK-NEXT: st1w { z7.s }, p0, [x9] -; CHECK-NEXT: st1w { z5.s }, p0, [x9] -; CHECK-NEXT: st1w { z4.s }, p0, [x9] -; CHECK-NEXT: st1w { z2.s }, p0, [x9] -; CHECK-NEXT: st1w { z1.s }, p0, [x9] +; CHECK-NEXT: ldr x8, [sp, #16] +; CHECK-NEXT: st1w { z1.s }, p0, [x8] +; CHECK-NEXT: st1w { z3.s }, p0, [x8] +; CHECK-NEXT: st1w { z6.s }, p0, [x8] +; CHECK-NEXT: st1w { z24.s }, p0, [x8] +; CHECK-NEXT: st1w { z7.s }, p0, [x8] +; CHECK-NEXT: st1w { z5.s }, p0, [x8] +; CHECK-NEXT: st1w { z4.s }, p0, [x8] +; CHECK-NEXT: st1w { z2.s }, p0, [x8] +; CHECK-NEXT: st1w { z0.s }, p0, [x8] ; CHECK-NEXT: ret entry: store volatile %s8, ptr %ptr @@ -339,8 +339,8 @@ entry: define void @aavpcs4(i32 %s0, i32 %s1, i32 %s2, i32 %s3, i32 %s4, i32 %s5, i32 %s6, i32 %s7, %s8, %s9, %s10, %s11, %s12, %s13, %s14, %s15, %s16, %s17, ptr %ptr) nounwind { ; CHECK-LABEL: aavpcs4: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldr x8, [sp] +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldr x9, [sp, #16] ; CHECK-NEXT: ld1w { z24.s }, p0/z, [x8] ; CHECK-NEXT: st1w { z0.s }, p0, [x9] @@ -371,11 +371,10 @@ entry: define @aavpcs5(float %s0, float %s1, float %s2, float %s3, float %s4, float %s5, float %s6, float %s7, %s8, %s9, %s10, %s11, %s12, %s13, %s14, %s15, %s16, %s17, ptr %ptr) nounwind { ; CHECK-LABEL: aavpcs5: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldr x8, [sp] -; CHECK-NEXT: ldr x9, [sp, #16] -; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0] +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ld1w { z1.s }, p0/z, [x8] +; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0] ; CHECK-NEXT: ld1w { z2.s }, p0/z, [x7] ; CHECK-NEXT: ld1w { z3.s }, p0/z, [x1] ; CHECK-NEXT: ld1w { z4.s }, p0/z, [x6] @@ -383,15 +382,16 @@ define @aavpcs5(float %s0, float %s1, float %s2, float %s3, ; CHECK-NEXT: ld1w { z6.s }, p0/z, [x2] ; CHECK-NEXT: ld1w { z7.s }, p0/z, [x4] ; CHECK-NEXT: ld1w { z24.s }, p0/z, [x3] -; CHECK-NEXT: st1w { z0.s }, p0, [x9] -; CHECK-NEXT: st1w { z3.s }, p0, [x9] -; CHECK-NEXT: st1w { z6.s }, p0, [x9] -; CHECK-NEXT: st1w { z24.s }, p0, [x9] -; CHECK-NEXT: st1w { z7.s }, p0, [x9] -; CHECK-NEXT: st1w { z5.s }, p0, [x9] -; CHECK-NEXT: st1w { z4.s }, p0, [x9] -; CHECK-NEXT: st1w { z2.s }, p0, [x9] -; CHECK-NEXT: st1w { z1.s }, p0, [x9] +; CHECK-NEXT: ldr x8, [sp, #16] +; CHECK-NEXT: st1w { z0.s }, p0, [x8] +; CHECK-NEXT: st1w { z3.s }, p0, [x8] +; CHECK-NEXT: st1w { z6.s }, p0, [x8] +; CHECK-NEXT: st1w { z24.s }, p0, [x8] +; CHECK-NEXT: st1w { z7.s }, p0, [x8] +; CHECK-NEXT: st1w { z5.s }, p0, [x8] +; CHECK-NEXT: st1w { z4.s }, p0, [x8] +; CHECK-NEXT: st1w { z2.s }, p0, [x8] +; CHECK-NEXT: st1w { z1.s }, p0, [x8] ; CHECK-NEXT: ret entry: store volatile %s8, ptr %ptr @@ -409,11 +409,10 @@ entry: define void @aapcs1(float %s0, float %s1, float %s2, float %s3, float %s4, float %s5, float %s6, float %s7, %s8, %s9, %s10, %s11, %s12, %s13, %s14, %s15, %s16, %s17, ptr %ptr) nounwind { ; CHECK-LABEL: aapcs1: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldr x8, [sp] -; CHECK-NEXT: ldr x9, [sp, #16] -; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0] -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x8] +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: ld1w { z0.s }, p0/z, [x8] +; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] ; CHECK-NEXT: ld1w { z2.s }, p0/z, [x7] ; CHECK-NEXT: ld1w { z3.s }, p0/z, [x1] ; CHECK-NEXT: ld1w { z4.s }, p0/z, [x6] @@ -421,15 +420,16 @@ define void @aapcs1(float %s0, float %s1, float %s2, float %s3, float %s4, float ; CHECK-NEXT: ld1w { z6.s }, p0/z, [x2] ; CHECK-NEXT: ld1w { z7.s }, p0/z, [x4] ; CHECK-NEXT: ld1w { z16.s }, p0/z, [x3] -; CHECK-NEXT: st1w { z0.s }, p0, [x9] -; CHECK-NEXT: st1w { z3.s }, p0, [x9] -; CHECK-NEXT: st1w { z6.s }, p0, [x9] -; CHECK-NEXT: st1w { z16.s }, p0, [x9] -; CHECK-NEXT: st1w { z7.s }, p0, [x9] -; CHECK-NEXT: st1w { z5.s }, p0, [x9] -; CHECK-NEXT: st1w { z4.s }, p0, [x9] -; CHECK-NEXT: st1w { z2.s }, p0, [x9] -; CHECK-NEXT: st1w { z1.s }, p0, [x9] +; CHECK-NEXT: ldr x8, [sp, #16] +; CHECK-NEXT: st1w { z1.s }, p0, [x8] +; CHECK-NEXT: st1w { z3.s }, p0, [x8] +; CHECK-NEXT: st1w { z6.s }, p0, [x8] +; CHECK-NEXT: st1w { z16.s }, p0, [x8] +; CHECK-NEXT: st1w { z7.s }, p0, [x8] +; CHECK-NEXT: st1w { z5.s }, p0, [x8] +; CHECK-NEXT: st1w { z4.s }, p0, [x8] +; CHECK-NEXT: st1w { z2.s }, p0, [x8] +; CHECK-NEXT: st1w { z0.s }, p0, [x8] ; CHECK-NEXT: ret entry: store volatile %s8, ptr %ptr @@ -486,13 +486,13 @@ define void @non_sve_caller_high_range_non_sve_callee_high_range(float %f0, floa ; CHECK-NEXT: fmov s2, #2.00000000 ; CHECK-NEXT: fmov s3, #3.00000000 ; CHECK-NEXT: fmov s4, #4.00000000 -; CHECK-NEXT: fmov s5, #5.00000000 -; CHECK-NEXT: fmov s6, #6.00000000 -; CHECK-NEXT: fmov s7, #7.00000000 ; CHECK-NEXT: ld1w { z16.s }, p0/z, [x0] ; CHECK-NEXT: ld1w { z17.s }, p0/z, [x1] ; CHECK-NEXT: addvl x0, sp, #1 +; CHECK-NEXT: fmov s5, #5.00000000 +; CHECK-NEXT: fmov s6, #6.00000000 ; CHECK-NEXT: mov x1, sp +; CHECK-NEXT: fmov s7, #7.00000000 ; CHECK-NEXT: st1w { z17.s }, p0, [sp] ; CHECK-NEXT: st1w { z16.s }, p0, [sp, #1, mul vl] ; CHECK-NEXT: bl non_sve_callee_high_range @@ -548,20 +548,20 @@ define @sve_caller_non_sve_callee_high_range( %a, %b) { ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov x8, #-1 // =0xffffffffffffffff -; CHECK-NEXT: whilels p1.s, xzr, x8 ; CHECK-NEXT: fcmeq p0.s, p0/z, z0.s, z1.s ; CHECK-NEXT: mov z0.s, p0/z, #1 // =0x1 -; CHECK-NEXT: lastb w8, p1, z0.s +; CHECK-NEXT: whilels p0.s, xzr, x8 +; CHECK-NEXT: lastb w8, p0, z0.s ; CHECK-NEXT: and w0, w8, #0x1 ; CHECK-NEXT: ret %vcond = fcmp oeq %a, %b diff --git a/llvm/test/CodeGen/AArch64/sve-doublereduct.ll b/llvm/test/CodeGen/AArch64/sve-doublereduct.ll index f5721cd0fd79..7bc31d44bb65 100644 --- a/llvm/test/CodeGen/AArch64/sve-doublereduct.ll +++ b/llvm/test/CodeGen/AArch64/sve-doublereduct.ll @@ -87,8 +87,8 @@ define float @fmaximum_f32( %a, %b) { define i32 @add_i32( %a, %b) { ; CHECK-LABEL: add_i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: add z0.s, z0.s, z1.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: add z0.s, z0.s, z2.s ; CHECK-NEXT: uaddv d0, p0, z0.s ; CHECK-NEXT: fmov x0, d0 @@ -160,8 +160,8 @@ define i16 @add_ext_v32i16( %a, %b) { define i32 @and_i32( %a, %b) { ; CHECK-LABEL: and_i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and z0.d, z0.d, z1.d +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and z0.d, z0.d, z2.d ; CHECK-NEXT: andv s0, p0, z0.s ; CHECK-NEXT: fmov w0, s0 @@ -175,8 +175,8 @@ define i32 @and_i32( %a, %b) { define i32 @or_i32( %a, %b) { ; CHECK-LABEL: or_i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: orr z0.d, z0.d, z1.d +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: orr z0.d, z0.d, z2.d ; CHECK-NEXT: orv s0, p0, z0.s ; CHECK-NEXT: fmov w0, s0 @@ -190,8 +190,8 @@ define i32 @or_i32( %a, %b) { define i32 @xor_i32( %a, %b) { ; CHECK-LABEL: xor_i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: eor3 z0.d, z0.d, z1.d, z2.d +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: eorv s0, p0, z0.s ; CHECK-NEXT: fmov w0, s0 ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sve-expand-div.ll b/llvm/test/CodeGen/AArch64/sve-expand-div.ll index fe5cdc938772..180c64e0a7de 100644 --- a/llvm/test/CodeGen/AArch64/sve-expand-div.ll +++ b/llvm/test/CodeGen/AArch64/sve-expand-div.ll @@ -10,8 +10,8 @@ define @sdiv_i8( %a) #0 { ; CHECK-LABEL: sdiv_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z1.b, #86 // =0x56 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: smulh z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: lsr z1.b, z0.b, #7 ; CHECK-NEXT: add z0.b, z0.b, z1.b @@ -23,8 +23,8 @@ define @sdiv_i8( %a) #0 { define @sdiv_i16( %a) #0 { ; CHECK-LABEL: sdiv_i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w8, #21846 // =0x5556 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: smulh z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: lsr z1.h, z0.h, #15 @@ -37,8 +37,8 @@ define @sdiv_i16( %a) #0 { define @sdiv_i32( %a) #0 { ; CHECK-LABEL: sdiv_i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #21846 // =0x5556 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: movk w8, #21845, lsl #16 ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: smulh z0.s, p0/m, z0.s, z1.s @@ -52,8 +52,8 @@ define @sdiv_i32( %a) #0 { define @sdiv_i64( %a) #0 { ; CHECK-LABEL: sdiv_i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, #6148914691236517205 // =0x5555555555555555 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: movk x8, #21846 ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: smulh z0.d, p0/m, z0.d, z1.d @@ -71,8 +71,8 @@ define @sdiv_i64( %a) #0 { define @udiv_i8( %a) #0 { ; CHECK-LABEL: udiv_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z1.b, #-85 // =0xffffffffffffffab +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: umulh z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: lsr z0.b, z0.b, #1 ; CHECK-NEXT: ret @@ -83,8 +83,8 @@ define @udiv_i8( %a) #0 { define @udiv_i16( %a) #0 { ; CHECK-LABEL: udiv_i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w8, #-21845 // =0xffffaaab +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: umulh z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: lsr z0.h, z0.h, #1 @@ -96,8 +96,8 @@ define @udiv_i16( %a) #0 { define @udiv_i32( %a) #0 { ; CHECK-LABEL: udiv_i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #43691 // =0xaaab +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: movk w8, #43690, lsl #16 ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: umulh z0.s, p0/m, z0.s, z1.s @@ -110,8 +110,8 @@ define @udiv_i32( %a) #0 { define @udiv_i64( %a) #0 { ; CHECK-LABEL: udiv_i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, #-6148914691236517206 // =0xaaaaaaaaaaaaaaaa +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: movk x8, #43691 ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: umulh z0.d, p0/m, z0.d, z1.d diff --git a/llvm/test/CodeGen/AArch64/sve-extract-element.ll b/llvm/test/CodeGen/AArch64/sve-extract-element.ll index a3c34b53baa0..6d4f5963881e 100644 --- a/llvm/test/CodeGen/AArch64/sve-extract-element.ll +++ b/llvm/test/CodeGen/AArch64/sve-extract-element.ll @@ -616,8 +616,8 @@ define i1 @test_last_8xi1( %a) #0 { ; CHECK: // %bb.0: ; CHECK-NEXT: mov x8, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z0.h, p0/z, #1 // =0x1 -; CHECK-NEXT: whilels p1.h, xzr, x8 -; CHECK-NEXT: lastb w8, p1, z0.h +; CHECK-NEXT: whilels p0.h, xzr, x8 +; CHECK-NEXT: lastb w8, p0, z0.h ; CHECK-NEXT: and w0, w8, #0x1 ; CHECK-NEXT: ret %vscale = call i64 @llvm.vscale.i64() @@ -630,10 +630,10 @@ define i1 @test_last_8xi1( %a) #0 { define i1 @test_lanex_4xi1( %a, i32 %x) #0 { ; CHECK-LABEL: test_lanex_4xi1: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, w0 ; CHECK-NEXT: mov z0.s, p0/z, #1 // =0x1 -; CHECK-NEXT: whilels p1.s, xzr, x8 -; CHECK-NEXT: lastb w8, p1, z0.s +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: whilels p0.s, xzr, x8 +; CHECK-NEXT: lastb w8, p0, z0.s ; CHECK-NEXT: and w0, w8, #0x1 ; CHECK-NEXT: ret %b = extractelement %a, i32 %x diff --git a/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll b/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll index bc1c563810f3..b9c531fe3352 100644 --- a/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll +++ b/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll @@ -100,16 +100,16 @@ define <2 x i64> @extract_v2i64_nxv8i64_8( %arg) { ; CHECK-NEXT: addvl sp, sp, #-4 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 32 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: cnth x8 ; CHECK-NEXT: mov w9, #8 // =0x8 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: sub x8, x8, #2 ; CHECK-NEXT: cmp x8, #8 +; CHECK-NEXT: st1d { z3.d }, p0, [sp, #3, mul vl] ; CHECK-NEXT: csel x8, x8, x9, lo +; CHECK-NEXT: st1d { z2.d }, p0, [sp, #2, mul vl] ; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: lsl x8, x8, #3 -; CHECK-NEXT: st1d { z3.d }, p0, [sp, #3, mul vl] -; CHECK-NEXT: st1d { z2.d }, p0, [sp, #2, mul vl] ; CHECK-NEXT: st1d { z1.d }, p0, [sp, #1, mul vl] ; CHECK-NEXT: st1d { z0.d }, p0, [sp] ; CHECK-NEXT: ldr q0, [x9, x8] @@ -183,10 +183,10 @@ define <4 x i1> @extract_v4i1_nxv32i1_16( %arg) { ; CHECK-NEXT: addvl sp, sp, #-8 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xc0, 0x00, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 64 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p2.b ; CHECK-NEXT: mov z0.b, p1/z, #1 // =0x1 ; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1 ; CHECK-NEXT: mov x8, sp +; CHECK-NEXT: ptrue p2.b ; CHECK-NEXT: add x8, x8, #16 ; CHECK-NEXT: st1b { z0.b }, p2, [sp, #1, mul vl] ; CHECK-NEXT: st1b { z1.b }, p2, [sp] diff --git a/llvm/test/CodeGen/AArch64/sve-extract-fixed-vector.ll b/llvm/test/CodeGen/AArch64/sve-extract-fixed-vector.ll index e2f8dad03ef6..88268104889f 100644 --- a/llvm/test/CodeGen/AArch64/sve-extract-fixed-vector.ll +++ b/llvm/test/CodeGen/AArch64/sve-extract-fixed-vector.ll @@ -17,15 +17,15 @@ define <2 x i64> @extract_v2i64_nxv2i64_idx2( %vec) nounwind { ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: cntd x8 ; CHECK-NEXT: mov w9, #2 // =0x2 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: sub x8, x8, #2 ; CHECK-NEXT: cmp x8, #2 +; CHECK-NEXT: st1d { z0.d }, p0, [sp] ; CHECK-NEXT: csel x8, x8, x9, lo ; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: lsl x8, x8, #3 -; CHECK-NEXT: st1d { z0.d }, p0, [sp] ; CHECK-NEXT: ldr q0, [x9, x8] ; CHECK-NEXT: addvl sp, sp, #1 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload @@ -50,15 +50,15 @@ define <4 x i32> @extract_v4i32_nxv4i32_idx4( %vec) nounwind { ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: cntw x8 ; CHECK-NEXT: mov w9, #4 // =0x4 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sub x8, x8, #4 ; CHECK-NEXT: cmp x8, #4 +; CHECK-NEXT: st1w { z0.s }, p0, [sp] ; CHECK-NEXT: csel x8, x8, x9, lo ; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: lsl x8, x8, #2 -; CHECK-NEXT: st1w { z0.s }, p0, [sp] ; CHECK-NEXT: ldr q0, [x9, x8] ; CHECK-NEXT: addvl sp, sp, #1 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload @@ -115,15 +115,15 @@ define <8 x i16> @extract_v8i16_nxv8i16_idx8( %vec) nounwind { ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: cnth x8 ; CHECK-NEXT: mov w9, #8 // =0x8 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: sub x8, x8, #8 ; CHECK-NEXT: cmp x8, #8 +; CHECK-NEXT: st1h { z0.h }, p0, [sp] ; CHECK-NEXT: csel x8, x8, x9, lo ; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: lsl x8, x8, #1 -; CHECK-NEXT: st1h { z0.h }, p0, [sp] ; CHECK-NEXT: ldr q0, [x9, x8] ; CHECK-NEXT: addvl sp, sp, #1 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload @@ -214,14 +214,14 @@ define <16 x i8> @extract_v16i8_nxv16i8_idx16( %vec) nounwind ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: rdvl x8, #1 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov w9, #16 // =0x10 ; CHECK-NEXT: sub x8, x8, #16 ; CHECK-NEXT: cmp x8, #16 +; CHECK-NEXT: st1b { z0.b }, p0, [sp] ; CHECK-NEXT: csel x8, x8, x9, lo ; CHECK-NEXT: mov x9, sp -; CHECK-NEXT: st1b { z0.b }, p0, [sp] ; CHECK-NEXT: ldr q0, [x9, x8] ; CHECK-NEXT: addvl sp, sp, #1 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload diff --git a/llvm/test/CodeGen/AArch64/sve-extract-scalable-vector.ll b/llvm/test/CodeGen/AArch64/sve-extract-scalable-vector.ll index e60a2f142922..3c0bd501f45d 100644 --- a/llvm/test/CodeGen/AArch64/sve-extract-scalable-vector.ll +++ b/llvm/test/CodeGen/AArch64/sve-extract-scalable-vector.ll @@ -65,27 +65,25 @@ define @extract_nxv14i1_nxv28i1_14( %in) uw ; CHECK-NEXT: addvl sp, sp, #-1 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG ; CHECK-NEXT: punpkhi p2.h, p1.b -; CHECK-NEXT: str p6, [sp, #5, mul vl] // 2-byte Folded Spill +; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: punpklo p1.h, p1.b ; CHECK-NEXT: str p5, [sp, #6, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: punpkhi p0.h, p0.b -; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: punpklo p2.h, p2.b ; CHECK-NEXT: punpkhi p3.h, p1.b -; CHECK-NEXT: punpklo p1.h, p1.b -; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: punpkhi p4.h, p2.b ; CHECK-NEXT: punpklo p2.h, p2.b +; CHECK-NEXT: uzp1 p4.s, p4.s, p0.s +; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: punpkhi p5.h, p3.b -; CHECK-NEXT: punpklo p3.h, p3.b -; CHECK-NEXT: punpkhi p6.h, p1.b ; CHECK-NEXT: punpklo p1.h, p1.b ; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: uzp1 p2.s, p5.s, p2.s +; CHECK-NEXT: punpklo p3.h, p3.b +; CHECK-NEXT: punpkhi p5.h, p1.b +; CHECK-NEXT: punpklo p1.h, p1.b +; CHECK-NEXT: punpkhi p0.h, p0.b +; CHECK-NEXT: uzp1 p3.s, p5.s, p3.s ; CHECK-NEXT: ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: uzp1 p3.s, p6.s, p3.s -; CHECK-NEXT: ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: uzp1 p4.s, p4.s, p0.s ; CHECK-NEXT: uzp1 p0.s, p0.s, p1.s ; CHECK-NEXT: uzp1 p1.h, p2.h, p4.h ; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload diff --git a/llvm/test/CodeGen/AArch64/sve-fcmp.ll b/llvm/test/CodeGen/AArch64/sve-fcmp.ll index f7e3b6d0171a..35cbe65c6a8b 100644 --- a/llvm/test/CodeGen/AArch64/sve-fcmp.ll +++ b/llvm/test/CodeGen/AArch64/sve-fcmp.ll @@ -374,8 +374,8 @@ define @one_zero( %x) { define @ueq_zero( %x) { ; CHECK-LABEL: ueq_zero: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, #0 // =0x0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: fcmuo p1.s, p0/z, z0.s, z1.s ; CHECK-NEXT: fcmeq p0.s, p0/z, z0.s, #0.0 ; CHECK-NEXT: sel p0.b, p0, p0.b, p1.b diff --git a/llvm/test/CodeGen/AArch64/sve-fcopysign.ll b/llvm/test/CodeGen/AArch64/sve-fcopysign.ll index f15807597ac2..78843e392e53 100644 --- a/llvm/test/CodeGen/AArch64/sve-fcopysign.ll +++ b/llvm/test/CodeGen/AArch64/sve-fcopysign.ll @@ -63,10 +63,10 @@ define @test_copysign_v4f32_v4f64( %a, ; CHECK-EXTEND-ROUND-NEXT: ptrue p0.d ; CHECK-EXTEND-ROUND-NEXT: uunpkhi z3.d, z0.s ; CHECK-EXTEND-ROUND-NEXT: uunpklo z0.d, z0.s -; CHECK-EXTEND-ROUND-NEXT: and z3.s, z3.s, #0x7fffffff -; CHECK-EXTEND-ROUND-NEXT: and z0.s, z0.s, #0x7fffffff ; CHECK-EXTEND-ROUND-NEXT: fcvt z2.s, p0/m, z2.d ; CHECK-EXTEND-ROUND-NEXT: fcvt z1.s, p0/m, z1.d +; CHECK-EXTEND-ROUND-NEXT: and z3.s, z3.s, #0x7fffffff +; CHECK-EXTEND-ROUND-NEXT: and z0.s, z0.s, #0x7fffffff ; CHECK-EXTEND-ROUND-NEXT: and z2.s, z2.s, #0x80000000 ; CHECK-EXTEND-ROUND-NEXT: and z1.s, z1.s, #0x80000000 ; CHECK-EXTEND-ROUND-NEXT: orr z2.d, z3.d, z2.d @@ -115,9 +115,9 @@ declare @llvm.copysign.v2f64( %a, @test_copysign_v4f64_v4f32( %a, %b) #0 { ; CHECK-LABEL: test_copysign_v4f64_v4f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: uunpklo z3.d, z2.s ; CHECK-NEXT: uunpkhi z2.d, z2.s +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and z0.d, z0.d, #0x7fffffffffffffff ; CHECK-NEXT: and z1.d, z1.d, #0x7fffffffffffffff ; CHECK-NEXT: fcvt z3.d, p0/m, z3.s @@ -193,10 +193,10 @@ define @test_copysign_v4f16_v4f64( %a, @test_copysign_v8f16_v8f32( %a, @test_copysign_v8f16_v8f32( %a, @test_copysign_nxv4f32_nxv4f16( %a, %b) #0 { ; CHECK-NO-EXTEND-ROUND-LABEL: test_copysign_nxv4f32_nxv4f16: ; CHECK-NO-EXTEND-ROUND: // %bb.0: -; CHECK-NO-EXTEND-ROUND-NEXT: ptrue p0.s ; CHECK-NO-EXTEND-ROUND-NEXT: and z1.s, z1.s, #0x80000000 ; CHECK-NO-EXTEND-ROUND-NEXT: and z0.s, z0.s, #0x7fffffff +; CHECK-NO-EXTEND-ROUND-NEXT: ptrue p0.s ; CHECK-NO-EXTEND-ROUND-NEXT: orr z0.d, z0.d, z1.d ; CHECK-NO-EXTEND-ROUND-NEXT: fcvt z0.h, p0/m, z0.s ; CHECK-NO-EXTEND-ROUND-NEXT: ret @@ -285,9 +285,9 @@ define @test_copysign_nxv4f32_nxv4f16( % define @test_copysign_nxv2f64_nxv2f32( %a, %b) #0 { ; CHECK-NO-EXTEND-ROUND-LABEL: test_copysign_nxv2f64_nxv2f32: ; CHECK-NO-EXTEND-ROUND: // %bb.0: -; CHECK-NO-EXTEND-ROUND-NEXT: ptrue p0.d ; CHECK-NO-EXTEND-ROUND-NEXT: and z1.d, z1.d, #0x8000000000000000 ; CHECK-NO-EXTEND-ROUND-NEXT: and z0.d, z0.d, #0x7fffffffffffffff +; CHECK-NO-EXTEND-ROUND-NEXT: ptrue p0.d ; CHECK-NO-EXTEND-ROUND-NEXT: orr z0.d, z0.d, z1.d ; CHECK-NO-EXTEND-ROUND-NEXT: fcvt z0.s, p0/m, z0.d ; CHECK-NO-EXTEND-ROUND-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sve-fcvt.ll b/llvm/test/CodeGen/AArch64/sve-fcvt.ll index 0fe38bf9ae71..fc5128fffad3 100644 --- a/llvm/test/CodeGen/AArch64/sve-fcvt.ll +++ b/llvm/test/CodeGen/AArch64/sve-fcvt.ll @@ -454,9 +454,9 @@ define @fcvtzu_d_nxv2f64( %a) { define @scvtf_h_nxv2i1( %a) { ; CHECK-LABEL: scvtf_h_nxv2i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: mov z0.d, p0/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: scvtf z0.h, p1/m, z0.d +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: scvtf z0.h, p0/m, z0.d ; CHECK-NEXT: ret %res = sitofp %a to ret %res @@ -495,9 +495,9 @@ define @scvtf_h_nxv2i64( %a) { define @scvtf_h_nxv3i1( %a) { ; CHECK-LABEL: scvtf_h_nxv3i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: mov z0.s, p0/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: scvtf z0.h, p1/m, z0.s +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: scvtf z0.h, p0/m, z0.s ; CHECK-NEXT: ret %res = sitofp %a to ret %res @@ -516,9 +516,9 @@ define @scvtf_h_nxv3i16( %a) { define @scvtf_h_nxv4i1( %a) { ; CHECK-LABEL: scvtf_h_nxv4i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: mov z0.s, p0/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: scvtf z0.h, p1/m, z0.s +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: scvtf z0.h, p0/m, z0.s ; CHECK-NEXT: ret %res = sitofp %a to ret %res @@ -547,9 +547,9 @@ define @scvtf_h_nxv4i32( %a) { define @scvtf_h_nxv7i1( %a) { ; CHECK-LABEL: scvtf_h_nxv7i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.h ; CHECK-NEXT: mov z0.h, p0/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: scvtf z0.h, p1/m, z0.h +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: scvtf z0.h, p0/m, z0.h ; CHECK-NEXT: ret %res = sitofp %a to ret %res @@ -568,9 +568,9 @@ define @scvtf_h_nxv7i16( %a) { define @scvtf_h_nxv8i1( %a) { ; CHECK-LABEL: scvtf_h_nxv8i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.h ; CHECK-NEXT: mov z0.h, p0/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: scvtf z0.h, p1/m, z0.h +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: scvtf z0.h, p0/m, z0.h ; CHECK-NEXT: ret %res = sitofp %a to ret %res @@ -589,9 +589,9 @@ define @scvtf_h_nxv8i16( %a) { define @scvtf_s_nxv2i1( %a) { ; CHECK-LABEL: scvtf_s_nxv2i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: mov z0.d, p0/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: scvtf z0.s, p1/m, z0.d +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: scvtf z0.s, p0/m, z0.d ; CHECK-NEXT: ret %res = sitofp %a to ret %res @@ -620,9 +620,9 @@ define @scvtf_s_nxv2i64( %a) { define @scvtf_s_nxv3i1( %a) { ; CHECK-LABEL: scvtf_s_nxv3i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: mov z0.s, p0/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: scvtf z0.s, p1/m, z0.s +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: scvtf z0.s, p0/m, z0.s ; CHECK-NEXT: ret %res = sitofp %a to ret %res @@ -641,9 +641,9 @@ define @scvtf_s_nxv3i32( %a) { define @scvtf_s_nxv4i1( %a) { ; CHECK-LABEL: scvtf_s_nxv4i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: mov z0.s, p0/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: scvtf z0.s, p1/m, z0.s +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: scvtf z0.s, p0/m, z0.s ; CHECK-NEXT: ret %res = sitofp %a to ret %res @@ -662,9 +662,9 @@ define @scvtf_s_nxv4i32( %a) { define @scvtf_d_nxv2i1( %a) { ; CHECK-LABEL: scvtf_d_nxv2i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: mov z0.d, p0/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: scvtf z0.d, p1/m, z0.d +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: scvtf z0.d, p0/m, z0.d ; CHECK-NEXT: ret %res = sitofp %a to ret %res @@ -695,9 +695,9 @@ define @scvtf_d_nxv2i64( %a) { define @ucvtf_h_nxv2i1( %a) { ; CHECK-LABEL: ucvtf_h_nxv2i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: mov z0.d, p0/z, #1 // =0x1 -; CHECK-NEXT: ucvtf z0.h, p1/m, z0.d +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: ucvtf z0.h, p0/m, z0.d ; CHECK-NEXT: ret %res = uitofp %a to ret %res @@ -736,9 +736,9 @@ define @ucvtf_h_nxv2i64( %a) { define @ucvtf_h_nxv3i1( %a) { ; CHECK-LABEL: ucvtf_h_nxv3i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: mov z0.s, p0/z, #1 // =0x1 -; CHECK-NEXT: ucvtf z0.h, p1/m, z0.s +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: ucvtf z0.h, p0/m, z0.s ; CHECK-NEXT: ret %res = uitofp %a to ret %res @@ -767,9 +767,9 @@ define @ucvtf_h_nxv3i32( %a) { define @ucvtf_h_nxv4i1( %a) { ; CHECK-LABEL: ucvtf_h_nxv4i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: mov z0.s, p0/z, #1 // =0x1 -; CHECK-NEXT: ucvtf z0.h, p1/m, z0.s +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: ucvtf z0.h, p0/m, z0.s ; CHECK-NEXT: ret %res = uitofp %a to ret %res @@ -798,9 +798,9 @@ define @ucvtf_h_nxv4i32( %a) { define @ucvtf_h_nxv8i1( %a) { ; CHECK-LABEL: ucvtf_h_nxv8i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.h ; CHECK-NEXT: mov z0.h, p0/z, #1 // =0x1 -; CHECK-NEXT: ucvtf z0.h, p1/m, z0.h +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: ucvtf z0.h, p0/m, z0.h ; CHECK-NEXT: ret %res = uitofp %a to ret %res @@ -819,9 +819,9 @@ define @ucvtf_h_nxv8i16( %a) { define @ucvtf_s_nxv2i1( %a) { ; CHECK-LABEL: ucvtf_s_nxv2i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: mov z0.d, p0/z, #1 // =0x1 -; CHECK-NEXT: ucvtf z0.s, p1/m, z0.d +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: ucvtf z0.s, p0/m, z0.d ; CHECK-NEXT: ret %res = uitofp %a to ret %res @@ -850,9 +850,9 @@ define @ucvtf_s_nxv2i64( %a) { define @ucvtf_s_nxv4i1( %a) { ; CHECK-LABEL: ucvtf_s_nxv4i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: mov z0.s, p0/z, #1 // =0x1 -; CHECK-NEXT: ucvtf z0.s, p1/m, z0.s +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: ucvtf z0.s, p0/m, z0.s ; CHECK-NEXT: ret %res = uitofp %a to ret %res @@ -871,9 +871,9 @@ define @ucvtf_s_nxv4i32( %a) { define @ucvtf_d_nxv2i1( %a) { ; CHECK-LABEL: ucvtf_d_nxv2i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: mov z0.d, p0/z, #1 // =0x1 -; CHECK-NEXT: ucvtf z0.d, p1/m, z0.d +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: ucvtf z0.d, p0/m, z0.d ; CHECK-NEXT: ret %res = uitofp %a to ret %res diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-addressing-modes.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-addressing-modes.ll index ed7ea657874a..28e1412c524a 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-addressing-modes.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-addressing-modes.ll @@ -7,8 +7,8 @@ target triple = "aarch64-unknown-linux-gnu" define void @masked_gather_base_plus_stride_v8f32(ptr %dst, ptr %src) #0 { ; CHECK-LABEL: masked_gather_base_plus_stride_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: index z0.s, #0, #7 +; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: ld1w { z0.s }, p0/z, [x1, z0.s, sxtw #2] ; CHECK-NEXT: st1w { z0.s }, p0, [x0] ; CHECK-NEXT: ret @@ -21,8 +21,8 @@ define void @masked_gather_base_plus_stride_v8f32(ptr %dst, ptr %src) #0 { define void @masked_gather_base_plus_stride_v4f64(ptr %dst, ptr %src) #0 { ; CHECK-LABEL: masked_gather_base_plus_stride_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: mov x8, #-32 // =0xffffffffffffffe0 +; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: index z0.d, #-2, x8 ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1, z0.d, lsl #3] ; CHECK-NEXT: st1d { z0.d }, p0, [x0] diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-build-vector.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-build-vector.ll index ad482118ec0b..47fda39d8400 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-build-vector.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-build-vector.ll @@ -7,8 +7,8 @@ target triple = "aarch64-unknown-linux-gnu" define void @build_vector_7_inc1_v32i8(ptr %a) #0 { ; VBITS_GE_256-LABEL: build_vector_7_inc1_v32i8: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.b, vl32 ; VBITS_GE_256-NEXT: index z0.b, #7, #1 +; VBITS_GE_256-NEXT: ptrue p0.b, vl32 ; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0] ; VBITS_GE_256-NEXT: ret store <32 x i8> , ptr %a, align 1 @@ -18,8 +18,8 @@ define void @build_vector_7_inc1_v32i8(ptr %a) #0 { define void @build_vector_0_inc2_v16i16(ptr %a) #0 { ; VBITS_GE_256-LABEL: build_vector_0_inc2_v16i16: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.h, vl16 ; VBITS_GE_256-NEXT: index z0.h, #0, #2 +; VBITS_GE_256-NEXT: ptrue p0.h, vl16 ; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0] ; VBITS_GE_256-NEXT: ret store <16 x i16> , ptr %a, align 2 @@ -30,8 +30,8 @@ define void @build_vector_0_inc2_v16i16(ptr %a) #0 { define void @build_vector_0_dec3_v8i32(ptr %a) #0 { ; VBITS_GE_256-LABEL: build_vector_0_dec3_v8i32: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: index z0.s, #0, #-3 +; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0] ; VBITS_GE_256-NEXT: ret store <8 x i32> , ptr %a, align 4 @@ -42,8 +42,8 @@ define void @build_vector_0_dec3_v8i32(ptr %a) #0 { define void @build_vector_minus2_dec32_v4i64(ptr %a) #0 { ; VBITS_GE_256-LABEL: build_vector_minus2_dec32_v4i64: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: mov x8, #-32 // =0xffffffffffffffe0 +; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: index z0.d, #-2, x8 ; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0] ; VBITS_GE_256-NEXT: ret @@ -53,11 +53,6 @@ define void @build_vector_minus2_dec32_v4i64(ptr %a) #0 { ; Constant but not a sequence. define void @build_vector_no_stride_v4i64(ptr %a) #0 { -; VBITS_GE_256-LABEL: .LCPI4_0: -; VBITS_GE_256: .xword 0 -; VBITS_GE_256-NEXT: .xword 4 -; VBITS_GE_256-NEXT: .xword 1 -; VBITS_GE_256-NEXT: .xword 8 ; VBITS_GE_256-LABEL: build_vector_no_stride_v4i64: ; VBITS_GE_256: // %bb.0: ; VBITS_GE_256-NEXT: ptrue p0.d, vl4 diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-concat.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-concat.ll index 65cb448cac11..f7751131005e 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-concat.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-concat.ll @@ -38,9 +38,9 @@ define void @concat_v32i8(ptr %a, ptr %b, ptr %c) vscale_range(2,0) #0 { ; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldr q0, [x0] ; CHECK-NEXT: ldr q1, [x1] -; CHECK-NEXT: ptrue p1.b, vl32 ; CHECK-NEXT: splice z0.b, p0, z0.b, z1.b -; CHECK-NEXT: st1b { z0.b }, p1, [x2] +; CHECK-NEXT: ptrue p0.b, vl32 +; CHECK-NEXT: st1b { z0.b }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <16 x i8>, ptr %a %op2 = load <16 x i8>, ptr %b @@ -66,11 +66,11 @@ define void @concat_v64i8(ptr %a, ptr %b, ptr %c) #0 { ; VBITS_GE_512-LABEL: concat_v64i8: ; VBITS_GE_512: // %bb.0: ; VBITS_GE_512-NEXT: ptrue p0.b, vl32 -; VBITS_GE_512-NEXT: ptrue p1.b, vl64 ; VBITS_GE_512-NEXT: ld1b { z0.b }, p0/z, [x0] ; VBITS_GE_512-NEXT: ld1b { z1.b }, p0/z, [x1] ; VBITS_GE_512-NEXT: splice z0.b, p0, z0.b, z1.b -; VBITS_GE_512-NEXT: st1b { z0.b }, p1, [x2] +; VBITS_GE_512-NEXT: ptrue p0.b, vl64 +; VBITS_GE_512-NEXT: st1b { z0.b }, p0, [x2] ; VBITS_GE_512-NEXT: ret %op1 = load <32 x i8>, ptr %a %op2 = load <32 x i8>, ptr %b @@ -90,11 +90,11 @@ define void @concat_v128i8(ptr %a, ptr %b, ptr %c) vscale_range(8,0) #0 { ; CHECK-LABEL: concat_v128i8: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.b, vl64 -; CHECK-NEXT: ptrue p1.b, vl128 ; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0] ; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1] ; CHECK-NEXT: splice z0.b, p0, z0.b, z1.b -; CHECK-NEXT: st1b { z0.b }, p1, [x2] +; CHECK-NEXT: ptrue p0.b, vl128 +; CHECK-NEXT: st1b { z0.b }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <64 x i8>, ptr %a %op2 = load <64 x i8>, ptr %b @@ -122,11 +122,11 @@ define void @concat_v256i8(ptr %a, ptr %b, ptr %c) vscale_range(16,0) #0 { ; CHECK-LABEL: concat_v256i8: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.b, vl128 -; CHECK-NEXT: ptrue p1.b, vl256 ; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0] ; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1] ; CHECK-NEXT: splice z0.b, p0, z0.b, z1.b -; CHECK-NEXT: st1b { z0.b }, p1, [x2] +; CHECK-NEXT: ptrue p0.b, vl256 +; CHECK-NEXT: st1b { z0.b }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <128 x i8>, ptr %a %op2 = load <128 x i8>, ptr %b @@ -198,9 +198,9 @@ define void @concat_v16i16(ptr %a, ptr %b, ptr %c) vscale_range(2,0) #0 { ; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldr q0, [x0] ; CHECK-NEXT: ldr q1, [x1] -; CHECK-NEXT: ptrue p1.h, vl16 ; CHECK-NEXT: splice z0.h, p0, z0.h, z1.h -; CHECK-NEXT: st1h { z0.h }, p1, [x2] +; CHECK-NEXT: ptrue p0.h, vl16 +; CHECK-NEXT: st1h { z0.h }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <8 x i16>, ptr %a %op2 = load <8 x i16>, ptr %b @@ -224,11 +224,11 @@ define void @concat_v32i16(ptr %a, ptr %b, ptr %c) #0 { ; VBITS_GE_512-LABEL: concat_v32i16: ; VBITS_GE_512: // %bb.0: ; VBITS_GE_512-NEXT: ptrue p0.h, vl16 -; VBITS_GE_512-NEXT: ptrue p1.h, vl32 ; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0] ; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x1] ; VBITS_GE_512-NEXT: splice z0.h, p0, z0.h, z1.h -; VBITS_GE_512-NEXT: st1h { z0.h }, p1, [x2] +; VBITS_GE_512-NEXT: ptrue p0.h, vl32 +; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x2] ; VBITS_GE_512-NEXT: ret %op1 = load <16 x i16>, ptr %a %op2 = load <16 x i16>, ptr %b @@ -244,11 +244,11 @@ define void @concat_v64i16(ptr %a, ptr %b, ptr %c) vscale_range(8,0) #0 { ; CHECK-LABEL: concat_v64i16: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.h, vl32 -; CHECK-NEXT: ptrue p1.h, vl64 ; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0] ; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1] ; CHECK-NEXT: splice z0.h, p0, z0.h, z1.h -; CHECK-NEXT: st1h { z0.h }, p1, [x2] +; CHECK-NEXT: ptrue p0.h, vl64 +; CHECK-NEXT: st1h { z0.h }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <32 x i16>, ptr %a %op2 = load <32 x i16>, ptr %b @@ -268,11 +268,11 @@ define void @concat_v128i16(ptr %a, ptr %b, ptr %c) vscale_range(16,0) #0 { ; CHECK-LABEL: concat_v128i16: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.h, vl64 -; CHECK-NEXT: ptrue p1.h, vl128 ; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0] ; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1] ; CHECK-NEXT: splice z0.h, p0, z0.h, z1.h -; CHECK-NEXT: st1h { z0.h }, p1, [x2] +; CHECK-NEXT: ptrue p0.h, vl128 +; CHECK-NEXT: st1h { z0.h }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <64 x i16>, ptr %a %op2 = load <64 x i16>, ptr %b @@ -328,9 +328,9 @@ define void @concat_v8i32(ptr %a, ptr %b, ptr %c) vscale_range(2,0) #0 { ; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldr q0, [x0] ; CHECK-NEXT: ldr q1, [x1] -; CHECK-NEXT: ptrue p1.s, vl8 ; CHECK-NEXT: splice z0.s, p0, z0.s, z1.s -; CHECK-NEXT: st1w { z0.s }, p1, [x2] +; CHECK-NEXT: ptrue p0.s, vl8 +; CHECK-NEXT: st1w { z0.s }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <4 x i32>, ptr %a %op2 = load <4 x i32>, ptr %b @@ -353,11 +353,11 @@ define void @concat_v16i32(ptr %a, ptr %b, ptr %c) #0 { ; VBITS_GE_512-LABEL: concat_v16i32: ; VBITS_GE_512: // %bb.0: ; VBITS_GE_512-NEXT: ptrue p0.s, vl8 -; VBITS_GE_512-NEXT: ptrue p1.s, vl16 ; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0] ; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x1] ; VBITS_GE_512-NEXT: splice z0.s, p0, z0.s, z1.s -; VBITS_GE_512-NEXT: st1w { z0.s }, p1, [x2] +; VBITS_GE_512-NEXT: ptrue p0.s, vl16 +; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x2] ; VBITS_GE_512-NEXT: ret %op1 = load <8 x i32>, ptr %a %op2 = load <8 x i32>, ptr %b @@ -371,11 +371,11 @@ define void @concat_v32i32(ptr %a, ptr %b, ptr %c) vscale_range(8,0) #0 { ; CHECK-LABEL: concat_v32i32: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.s, vl16 -; CHECK-NEXT: ptrue p1.s, vl32 ; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0] ; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1] ; CHECK-NEXT: splice z0.s, p0, z0.s, z1.s -; CHECK-NEXT: st1w { z0.s }, p1, [x2] +; CHECK-NEXT: ptrue p0.s, vl32 +; CHECK-NEXT: st1w { z0.s }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <16 x i32>, ptr %a %op2 = load <16 x i32>, ptr %b @@ -391,11 +391,11 @@ define void @concat_v64i32(ptr %a, ptr %b, ptr %c) vscale_range(16,0) #0 { ; CHECK-LABEL: concat_v64i32: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.s, vl32 -; CHECK-NEXT: ptrue p1.s, vl64 ; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0] ; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1] ; CHECK-NEXT: splice z0.s, p0, z0.s, z1.s -; CHECK-NEXT: st1w { z0.s }, p1, [x2] +; CHECK-NEXT: ptrue p0.s, vl64 +; CHECK-NEXT: st1w { z0.s }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <32 x i32>, ptr %a %op2 = load <32 x i32>, ptr %b @@ -433,9 +433,9 @@ define void @concat_v4i64(ptr %a, ptr %b, ptr %c) vscale_range(2,0) #0 { ; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldr q0, [x0] ; CHECK-NEXT: ldr q1, [x1] -; CHECK-NEXT: ptrue p1.d, vl4 ; CHECK-NEXT: splice z0.d, p0, z0.d, z1.d -; CHECK-NEXT: st1d { z0.d }, p1, [x2] +; CHECK-NEXT: ptrue p0.d, vl4 +; CHECK-NEXT: st1d { z0.d }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <2 x i64>, ptr %a %op2 = load <2 x i64>, ptr %b @@ -458,11 +458,11 @@ define void @concat_v8i64(ptr %a, ptr %b, ptr %c) #0 { ; VBITS_GE_512-LABEL: concat_v8i64: ; VBITS_GE_512: // %bb.0: ; VBITS_GE_512-NEXT: ptrue p0.d, vl4 -; VBITS_GE_512-NEXT: ptrue p1.d, vl8 ; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0] ; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1] ; VBITS_GE_512-NEXT: splice z0.d, p0, z0.d, z1.d -; VBITS_GE_512-NEXT: st1d { z0.d }, p1, [x2] +; VBITS_GE_512-NEXT: ptrue p0.d, vl8 +; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2] ; VBITS_GE_512-NEXT: ret %op1 = load <4 x i64>, ptr %a %op2 = load <4 x i64>, ptr %b @@ -475,11 +475,11 @@ define void @concat_v16i64(ptr %a, ptr %b, ptr %c) vscale_range(8,0) #0 { ; CHECK-LABEL: concat_v16i64: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.d, vl8 -; CHECK-NEXT: ptrue p1.d, vl16 ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] ; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] ; CHECK-NEXT: splice z0.d, p0, z0.d, z1.d -; CHECK-NEXT: st1d { z0.d }, p1, [x2] +; CHECK-NEXT: ptrue p0.d, vl16 +; CHECK-NEXT: st1d { z0.d }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <8 x i64>, ptr %a %op2 = load <8 x i64>, ptr %b @@ -493,11 +493,11 @@ define void @concat_v32i64(ptr %a, ptr %b, ptr %c) vscale_range(16,0) #0 { ; CHECK-LABEL: concat_v32i64: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.d, vl16 -; CHECK-NEXT: ptrue p1.d, vl32 ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] ; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] ; CHECK-NEXT: splice z0.d, p0, z0.d, z1.d -; CHECK-NEXT: st1d { z0.d }, p1, [x2] +; CHECK-NEXT: ptrue p0.d, vl32 +; CHECK-NEXT: st1d { z0.d }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <16 x i64>, ptr %a %op2 = load <16 x i64>, ptr %b @@ -541,9 +541,9 @@ define void @concat_v16f16(ptr %a, ptr %b, ptr %c) vscale_range(2,0) #0 { ; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldr q0, [x0] ; CHECK-NEXT: ldr q1, [x1] -; CHECK-NEXT: ptrue p1.h, vl16 ; CHECK-NEXT: splice z0.h, p0, z0.h, z1.h -; CHECK-NEXT: st1h { z0.h }, p1, [x2] +; CHECK-NEXT: ptrue p0.h, vl16 +; CHECK-NEXT: st1h { z0.h }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <8 x half>, ptr %a %op2 = load <8 x half>, ptr %b @@ -567,11 +567,11 @@ define void @concat_v32f16(ptr %a, ptr %b, ptr %c) #0 { ; VBITS_GE_512-LABEL: concat_v32f16: ; VBITS_GE_512: // %bb.0: ; VBITS_GE_512-NEXT: ptrue p0.h, vl16 -; VBITS_GE_512-NEXT: ptrue p1.h, vl32 ; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0] ; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x1] ; VBITS_GE_512-NEXT: splice z0.h, p0, z0.h, z1.h -; VBITS_GE_512-NEXT: st1h { z0.h }, p1, [x2] +; VBITS_GE_512-NEXT: ptrue p0.h, vl32 +; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x2] ; VBITS_GE_512-NEXT: ret %op1 = load <16 x half>, ptr %a %op2 = load <16 x half>, ptr %b @@ -587,11 +587,11 @@ define void @concat_v64f16(ptr %a, ptr %b, ptr %c) vscale_range(8,0) #0 { ; CHECK-LABEL: concat_v64f16: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.h, vl32 -; CHECK-NEXT: ptrue p1.h, vl64 ; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0] ; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1] ; CHECK-NEXT: splice z0.h, p0, z0.h, z1.h -; CHECK-NEXT: st1h { z0.h }, p1, [x2] +; CHECK-NEXT: ptrue p0.h, vl64 +; CHECK-NEXT: st1h { z0.h }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <32 x half>, ptr %a %op2 = load <32 x half>, ptr %b @@ -611,11 +611,11 @@ define void @concat_v128f16(ptr %a, ptr %b, ptr %c) vscale_range(16,0) #0 { ; CHECK-LABEL: concat_v128f16: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.h, vl64 -; CHECK-NEXT: ptrue p1.h, vl128 ; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0] ; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1] ; CHECK-NEXT: splice z0.h, p0, z0.h, z1.h -; CHECK-NEXT: st1h { z0.h }, p1, [x2] +; CHECK-NEXT: ptrue p0.h, vl128 +; CHECK-NEXT: st1h { z0.h }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <64 x half>, ptr %a %op2 = load <64 x half>, ptr %b @@ -671,9 +671,9 @@ define void @concat_v8f32(ptr %a, ptr %b, ptr %c) vscale_range(2,0) #0 { ; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldr q0, [x0] ; CHECK-NEXT: ldr q1, [x1] -; CHECK-NEXT: ptrue p1.s, vl8 ; CHECK-NEXT: splice z0.s, p0, z0.s, z1.s -; CHECK-NEXT: st1w { z0.s }, p1, [x2] +; CHECK-NEXT: ptrue p0.s, vl8 +; CHECK-NEXT: st1w { z0.s }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <4 x float>, ptr %a %op2 = load <4 x float>, ptr %b @@ -696,11 +696,11 @@ define void @concat_v16f32(ptr %a, ptr %b, ptr %c) #0 { ; VBITS_GE_512-LABEL: concat_v16f32: ; VBITS_GE_512: // %bb.0: ; VBITS_GE_512-NEXT: ptrue p0.s, vl8 -; VBITS_GE_512-NEXT: ptrue p1.s, vl16 ; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0] ; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x1] ; VBITS_GE_512-NEXT: splice z0.s, p0, z0.s, z1.s -; VBITS_GE_512-NEXT: st1w { z0.s }, p1, [x2] +; VBITS_GE_512-NEXT: ptrue p0.s, vl16 +; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x2] ; VBITS_GE_512-NEXT: ret %op1 = load <8 x float>, ptr %a %op2 = load <8 x float>, ptr %b @@ -714,11 +714,11 @@ define void @concat_v32f32(ptr %a, ptr %b, ptr %c) vscale_range(8,0) #0 { ; CHECK-LABEL: concat_v32f32: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.s, vl16 -; CHECK-NEXT: ptrue p1.s, vl32 ; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0] ; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1] ; CHECK-NEXT: splice z0.s, p0, z0.s, z1.s -; CHECK-NEXT: st1w { z0.s }, p1, [x2] +; CHECK-NEXT: ptrue p0.s, vl32 +; CHECK-NEXT: st1w { z0.s }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <16 x float>, ptr %a %op2 = load <16 x float>, ptr %b @@ -734,11 +734,11 @@ define void @concat_v64f32(ptr %a, ptr %b, ptr %c) vscale_range(16,0) #0 { ; CHECK-LABEL: concat_v64f32: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.s, vl32 -; CHECK-NEXT: ptrue p1.s, vl64 ; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0] ; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1] ; CHECK-NEXT: splice z0.s, p0, z0.s, z1.s -; CHECK-NEXT: st1w { z0.s }, p1, [x2] +; CHECK-NEXT: ptrue p0.s, vl64 +; CHECK-NEXT: st1w { z0.s }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <32 x float>, ptr %a %op2 = load <32 x float>, ptr %b @@ -776,9 +776,9 @@ define void @concat_v4f64(ptr %a, ptr %b, ptr %c) vscale_range(2,0) #0 { ; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldr q0, [x0] ; CHECK-NEXT: ldr q1, [x1] -; CHECK-NEXT: ptrue p1.d, vl4 ; CHECK-NEXT: splice z0.d, p0, z0.d, z1.d -; CHECK-NEXT: st1d { z0.d }, p1, [x2] +; CHECK-NEXT: ptrue p0.d, vl4 +; CHECK-NEXT: st1d { z0.d }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <2 x double>, ptr %a %op2 = load <2 x double>, ptr %b @@ -801,11 +801,11 @@ define void @concat_v8f64(ptr %a, ptr %b, ptr %c) #0 { ; VBITS_GE_512-LABEL: concat_v8f64: ; VBITS_GE_512: // %bb.0: ; VBITS_GE_512-NEXT: ptrue p0.d, vl4 -; VBITS_GE_512-NEXT: ptrue p1.d, vl8 ; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0] ; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1] ; VBITS_GE_512-NEXT: splice z0.d, p0, z0.d, z1.d -; VBITS_GE_512-NEXT: st1d { z0.d }, p1, [x2] +; VBITS_GE_512-NEXT: ptrue p0.d, vl8 +; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2] ; VBITS_GE_512-NEXT: ret %op1 = load <4 x double>, ptr %a %op2 = load <4 x double>, ptr %b @@ -818,11 +818,11 @@ define void @concat_v16f64(ptr %a, ptr %b, ptr %c) vscale_range(8,0) #0 { ; CHECK-LABEL: concat_v16f64: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.d, vl8 -; CHECK-NEXT: ptrue p1.d, vl16 ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] ; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] ; CHECK-NEXT: splice z0.d, p0, z0.d, z1.d -; CHECK-NEXT: st1d { z0.d }, p1, [x2] +; CHECK-NEXT: ptrue p0.d, vl16 +; CHECK-NEXT: st1d { z0.d }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <8 x double>, ptr %a %op2 = load <8 x double>, ptr %b @@ -836,11 +836,11 @@ define void @concat_v32f64(ptr %a, ptr %b, ptr %c) vscale_range(16,0) #0 { ; CHECK-LABEL: concat_v32f64: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.d, vl16 -; CHECK-NEXT: ptrue p1.d, vl32 ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] ; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] ; CHECK-NEXT: splice z0.d, p0, z0.d, z1.d -; CHECK-NEXT: st1d { z0.d }, p1, [x2] +; CHECK-NEXT: ptrue p0.d, vl32 +; CHECK-NEXT: st1d { z0.d }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <16 x double>, ptr %a %op2 = load <16 x double>, ptr %b diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-extract-vector-elt.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-extract-vector-elt.ll index 485124c1d59e..ad4efeaf3924 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-extract-vector-elt.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-extract-vector-elt.ll @@ -70,9 +70,9 @@ define half @extractelement_v64f16(ptr %a) vscale_range(8,0) #0 { ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.h, vl64 ; CHECK-NEXT: mov w8, #63 // =0x3f -; CHECK-NEXT: whilels p1.h, xzr, x8 ; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0] -; CHECK-NEXT: lastb h0, p1, z0.h +; CHECK-NEXT: whilels p0.h, xzr, x8 +; CHECK-NEXT: lastb h0, p0, z0.h ; CHECK-NEXT: ret %op1 = load <64 x half>, ptr %a %r = extractelement <64 x half> %op1, i64 63 @@ -84,9 +84,9 @@ define half @extractelement_v128f16(ptr %a) vscale_range(16,0) #0 { ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.h, vl128 ; CHECK-NEXT: mov w8, #127 // =0x7f -; CHECK-NEXT: whilels p1.h, xzr, x8 ; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0] -; CHECK-NEXT: lastb h0, p1, z0.h +; CHECK-NEXT: whilels p0.h, xzr, x8 +; CHECK-NEXT: lastb h0, p0, z0.h ; CHECK-NEXT: ret %op1 = load <128 x half>, ptr %a %r = extractelement <128 x half> %op1, i64 127 @@ -154,9 +154,9 @@ define float @extractelement_v32f32(ptr %a) vscale_range(8,0) #0 { ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.s, vl32 ; CHECK-NEXT: mov w8, #31 // =0x1f -; CHECK-NEXT: whilels p1.s, xzr, x8 ; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0] -; CHECK-NEXT: lastb s0, p1, z0.s +; CHECK-NEXT: whilels p0.s, xzr, x8 +; CHECK-NEXT: lastb s0, p0, z0.s ; CHECK-NEXT: ret %op1 = load <32 x float>, ptr %a %r = extractelement <32 x float> %op1, i64 31 @@ -168,9 +168,9 @@ define float @extractelement_v64f32(ptr %a) vscale_range(16,0) #0 { ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.s, vl64 ; CHECK-NEXT: mov w8, #63 // =0x3f -; CHECK-NEXT: whilels p1.s, xzr, x8 ; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0] -; CHECK-NEXT: lastb s0, p1, z0.s +; CHECK-NEXT: whilels p0.s, xzr, x8 +; CHECK-NEXT: lastb s0, p0, z0.s ; CHECK-NEXT: ret %op1 = load <64 x float>, ptr %a %r = extractelement <64 x float> %op1, i64 63 @@ -236,9 +236,9 @@ define double @extractelement_v16f64(ptr %a) vscale_range(8,0) #0 { ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.d, vl16 ; CHECK-NEXT: mov w8, #15 // =0xf -; CHECK-NEXT: whilels p1.d, xzr, x8 ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] -; CHECK-NEXT: lastb d0, p1, z0.d +; CHECK-NEXT: whilels p0.d, xzr, x8 +; CHECK-NEXT: lastb d0, p0, z0.d ; CHECK-NEXT: ret %op1 = load <16 x double>, ptr %a %r = extractelement <16 x double> %op1, i64 15 @@ -250,9 +250,9 @@ define double @extractelement_v32f64(ptr %a) vscale_range(16,0) #0 { ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.d, vl32 ; CHECK-NEXT: mov w8, #31 // =0x1f -; CHECK-NEXT: whilels p1.d, xzr, x8 ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] -; CHECK-NEXT: lastb d0, p1, z0.d +; CHECK-NEXT: whilels p0.d, xzr, x8 +; CHECK-NEXT: lastb d0, p0, z0.d ; CHECK-NEXT: ret %op1 = load <32 x double>, ptr %a %r = extractelement <32 x double> %op1, i64 31 diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-fcopysign.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-fcopysign.ll index bca3dfe5717e..e77cd9ef55ea 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-fcopysign.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-fcopysign.ll @@ -396,9 +396,9 @@ define void @test_copysign_v4f32_v4f64(ptr %ap, ptr %bp) vscale_range(2,0) #0 { ; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: mvni v1.4s, #128, lsl #24 ; CHECK-NEXT: ldr q2, [x0] -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1] -; CHECK-NEXT: fcvt z0.s, p1/m, z0.d +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: fcvt z0.s, p0/m, z0.d ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s ; CHECK-NEXT: bit v0.16b, v2.16b, v1.16b ; CHECK-NEXT: str q0, [x0] @@ -450,12 +450,12 @@ define void @test_copysign_v4f64_v4f32(ptr %ap, ptr %bp) vscale_range(2,0) #0 { ; ; CHECK_EXTEND_ROUND-LABEL: test_copysign_v4f64_v4f32: ; CHECK_EXTEND_ROUND: // %bb.0: -; CHECK_EXTEND_ROUND-NEXT: ptrue p0.d, vl4 ; CHECK_EXTEND_ROUND-NEXT: ldr q0, [x1] +; CHECK_EXTEND_ROUND-NEXT: ptrue p0.d, vl4 ; CHECK_EXTEND_ROUND-NEXT: uunpklo z0.d, z0.s -; CHECK_EXTEND_ROUND-NEXT: fcvt z0.d, p0/m, z0.s ; CHECK_EXTEND_ROUND-NEXT: ld1d { z1.d }, p0/z, [x0] ; CHECK_EXTEND_ROUND-NEXT: and z1.d, z1.d, #0x7fffffffffffffff +; CHECK_EXTEND_ROUND-NEXT: fcvt z0.d, p0/m, z0.s ; CHECK_EXTEND_ROUND-NEXT: and z0.d, z0.d, #0x8000000000000000 ; CHECK_EXTEND_ROUND-NEXT: orr z0.d, z1.d, z0.d ; CHECK_EXTEND_ROUND-NEXT: st1d { z0.d }, p0, [x0] @@ -494,9 +494,9 @@ define void @test_copysign_v4f16_v4f64(ptr %ap, ptr %bp) vscale_range(2,0) #0 { ; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: mvni v1.4h, #128, lsl #8 ; CHECK-NEXT: ldr d2, [x0] -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1] -; CHECK-NEXT: fcvt z0.h, p1/m, z0.d +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: fcvt z0.h, p0/m, z0.d ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s ; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: bit v0.8b, v2.8b, v1.8b @@ -521,9 +521,9 @@ define void @test_copysign_v8f16_v8f32(ptr %ap, ptr %bp) vscale_range(2,0) #0 { ; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: mvni v1.8h, #128, lsl #8 ; CHECK-NEXT: ldr q2, [x0] -; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: ld1w { z0.s }, p0/z, [x1] -; CHECK-NEXT: fcvt z0.h, p1/m, z0.s +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: fcvt z0.h, p0/m, z0.s ; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: bit v0.16b, v2.16b, v1.16b ; CHECK-NEXT: str q0, [x0] diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-extend-trunc.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-extend-trunc.ll index 6da07b855a5c..b60988be1e76 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-extend-trunc.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-extend-trunc.ll @@ -462,11 +462,11 @@ define void @fcvt_v8f64_v8f16(ptr %a, ptr %b) #0 { ; VBITS_GE_256: // %bb.0: ; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 -; VBITS_GE_256-NEXT: ptrue p1.d ; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] ; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0] -; VBITS_GE_256-NEXT: fcvt z0.h, p1/m, z0.d -; VBITS_GE_256-NEXT: fcvt z1.h, p1/m, z1.d +; VBITS_GE_256-NEXT: ptrue p0.d +; VBITS_GE_256-NEXT: fcvt z0.h, p0/m, z0.d +; VBITS_GE_256-NEXT: fcvt z1.h, p0/m, z1.d ; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s ; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-select.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-select.ll index 13ebda1df7f9..d1e9dc13f50e 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-select.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-select.ll @@ -34,15 +34,15 @@ define <8 x half> @select_v8f16(<8 x half> %op1, <8 x half> %op2, i1 %mask) vsca define void @select_v16f16(ptr %a, ptr %b, i1 %mask) vscale_range(2,0) #0 { ; CHECK-LABEL: select_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl16 ; CHECK-NEXT: mov z0.h, w2 -; CHECK-NEXT: ptrue p1.h +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: ptrue p1.h, vl16 ; CHECK-NEXT: and z0.h, z0.h, #0x1 -; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0] -; CHECK-NEXT: ld1h { z2.h }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.h, p1/z, z0.h, #0 -; CHECK-NEXT: sel z0.h, p1, z1.h, z2.h -; CHECK-NEXT: st1h { z0.h }, p0, [x0] +; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #0 +; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0] +; CHECK-NEXT: ld1h { z1.h }, p1/z, [x1] +; CHECK-NEXT: sel z0.h, p0, z0.h, z1.h +; CHECK-NEXT: st1h { z0.h }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <16 x half>, ptr %a %op2 = load volatile <16 x half>, ptr %b @@ -54,33 +54,33 @@ define void @select_v16f16(ptr %a, ptr %b, i1 %mask) vscale_range(2,0) #0 { define void @select_v32f16(ptr %a, ptr %b, i1 %mask) #0 { ; VBITS_GE_256-LABEL: select_v32f16: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.h, vl16 ; VBITS_GE_256-NEXT: mov z0.h, w2 +; VBITS_GE_256-NEXT: ptrue p0.h ; VBITS_GE_256-NEXT: mov x8, #16 // =0x10 -; VBITS_GE_256-NEXT: ptrue p1.h +; VBITS_GE_256-NEXT: ptrue p1.h, vl16 ; VBITS_GE_256-NEXT: and z0.h, z0.h, #0x1 -; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x0, x8, lsl #1] -; VBITS_GE_256-NEXT: ld1h { z2.h }, p0/z, [x0] -; VBITS_GE_256-NEXT: ld1h { z3.h }, p0/z, [x1, x8, lsl #1] -; VBITS_GE_256-NEXT: cmpne p1.h, p1/z, z0.h, #0 -; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x1] -; VBITS_GE_256-NEXT: sel z1.h, p1, z1.h, z3.h -; VBITS_GE_256-NEXT: mov z0.h, p1/m, z2.h -; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x0, x8, lsl #1] -; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0] +; VBITS_GE_256-NEXT: cmpne p0.h, p0/z, z0.h, #0 +; VBITS_GE_256-NEXT: ld1h { z0.h }, p1/z, [x0, x8, lsl #1] +; VBITS_GE_256-NEXT: ld1h { z1.h }, p1/z, [x0] +; VBITS_GE_256-NEXT: ld1h { z2.h }, p1/z, [x1, x8, lsl #1] +; VBITS_GE_256-NEXT: ld1h { z3.h }, p1/z, [x1] +; VBITS_GE_256-NEXT: sel z0.h, p0, z0.h, z2.h +; VBITS_GE_256-NEXT: sel z1.h, p0, z1.h, z3.h +; VBITS_GE_256-NEXT: st1h { z0.h }, p1, [x0, x8, lsl #1] +; VBITS_GE_256-NEXT: st1h { z1.h }, p1, [x0] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: select_v32f16: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.h, vl32 ; VBITS_GE_512-NEXT: mov z0.h, w2 -; VBITS_GE_512-NEXT: ptrue p1.h +; VBITS_GE_512-NEXT: ptrue p0.h +; VBITS_GE_512-NEXT: ptrue p1.h, vl32 ; VBITS_GE_512-NEXT: and z0.h, z0.h, #0x1 -; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x0] -; VBITS_GE_512-NEXT: ld1h { z2.h }, p0/z, [x1] -; VBITS_GE_512-NEXT: cmpne p1.h, p1/z, z0.h, #0 -; VBITS_GE_512-NEXT: sel z0.h, p1, z1.h, z2.h -; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0] +; VBITS_GE_512-NEXT: cmpne p0.h, p0/z, z0.h, #0 +; VBITS_GE_512-NEXT: ld1h { z0.h }, p1/z, [x0] +; VBITS_GE_512-NEXT: ld1h { z1.h }, p1/z, [x1] +; VBITS_GE_512-NEXT: sel z0.h, p0, z0.h, z1.h +; VBITS_GE_512-NEXT: st1h { z0.h }, p1, [x0] ; VBITS_GE_512-NEXT: ret %op1 = load volatile <32 x half>, ptr %a %op2 = load volatile <32 x half>, ptr %b @@ -92,15 +92,15 @@ define void @select_v32f16(ptr %a, ptr %b, i1 %mask) #0 { define void @select_v64f16(ptr %a, ptr %b, i1 %mask) vscale_range(8,0) #0 { ; CHECK-LABEL: select_v64f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl64 ; CHECK-NEXT: mov z0.h, w2 -; CHECK-NEXT: ptrue p1.h +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: ptrue p1.h, vl64 ; CHECK-NEXT: and z0.h, z0.h, #0x1 -; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0] -; CHECK-NEXT: ld1h { z2.h }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.h, p1/z, z0.h, #0 -; CHECK-NEXT: sel z0.h, p1, z1.h, z2.h -; CHECK-NEXT: st1h { z0.h }, p0, [x0] +; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #0 +; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0] +; CHECK-NEXT: ld1h { z1.h }, p1/z, [x1] +; CHECK-NEXT: sel z0.h, p0, z0.h, z1.h +; CHECK-NEXT: st1h { z0.h }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <64 x half>, ptr %a %op2 = load volatile <64 x half>, ptr %b @@ -112,15 +112,15 @@ define void @select_v64f16(ptr %a, ptr %b, i1 %mask) vscale_range(8,0) #0 { define void @select_v128f16(ptr %a, ptr %b, i1 %mask) vscale_range(16,0) #0 { ; CHECK-LABEL: select_v128f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl128 ; CHECK-NEXT: mov z0.h, w2 -; CHECK-NEXT: ptrue p1.h +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: ptrue p1.h, vl128 ; CHECK-NEXT: and z0.h, z0.h, #0x1 -; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0] -; CHECK-NEXT: ld1h { z2.h }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.h, p1/z, z0.h, #0 -; CHECK-NEXT: sel z0.h, p1, z1.h, z2.h -; CHECK-NEXT: st1h { z0.h }, p0, [x0] +; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #0 +; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0] +; CHECK-NEXT: ld1h { z1.h }, p1/z, [x1] +; CHECK-NEXT: sel z0.h, p0, z0.h, z1.h +; CHECK-NEXT: st1h { z0.h }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <128 x half>, ptr %a %op2 = load volatile <128 x half>, ptr %b @@ -158,15 +158,15 @@ define <4 x float> @select_v4f32(<4 x float> %op1, <4 x float> %op2, i1 %mask) v define void @select_v8f32(ptr %a, ptr %b, i1 %mask) vscale_range(2,0) #0 { ; CHECK-LABEL: select_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: and w8, w2, #0x1 -; CHECK-NEXT: ptrue p1.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z0.s, w8 -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.s, p1/z, z0.s, #0 -; CHECK-NEXT: sel z0.s, p1, z1.s, z2.s -; CHECK-NEXT: st1w { z0.s }, p0, [x0] +; CHECK-NEXT: ptrue p1.s, vl8 +; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0 +; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0] +; CHECK-NEXT: ld1w { z1.s }, p1/z, [x1] +; CHECK-NEXT: sel z0.s, p0, z0.s, z1.s +; CHECK-NEXT: st1w { z0.s }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <8 x float>, ptr %a %op2 = load volatile <8 x float>, ptr %b @@ -178,33 +178,33 @@ define void @select_v8f32(ptr %a, ptr %b, i1 %mask) vscale_range(2,0) #0 { define void @select_v16f32(ptr %a, ptr %b, i1 %mask) #0 { ; VBITS_GE_256-LABEL: select_v16f32: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: and w8, w2, #0x1 -; VBITS_GE_256-NEXT: ptrue p1.s +; VBITS_GE_256-NEXT: ptrue p0.s ; VBITS_GE_256-NEXT: mov z0.s, w8 +; VBITS_GE_256-NEXT: ptrue p1.s, vl8 ; VBITS_GE_256-NEXT: mov x8, #8 // =0x8 -; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0, x8, lsl #2] -; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x0] -; VBITS_GE_256-NEXT: ld1w { z3.s }, p0/z, [x1, x8, lsl #2] -; VBITS_GE_256-NEXT: cmpne p1.s, p1/z, z0.s, #0 -; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x1] -; VBITS_GE_256-NEXT: sel z1.s, p1, z1.s, z3.s -; VBITS_GE_256-NEXT: mov z0.s, p1/m, z2.s -; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0, x8, lsl #2] -; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0] +; VBITS_GE_256-NEXT: cmpne p0.s, p0/z, z0.s, #0 +; VBITS_GE_256-NEXT: ld1w { z0.s }, p1/z, [x0, x8, lsl #2] +; VBITS_GE_256-NEXT: ld1w { z1.s }, p1/z, [x0] +; VBITS_GE_256-NEXT: ld1w { z2.s }, p1/z, [x1, x8, lsl #2] +; VBITS_GE_256-NEXT: ld1w { z3.s }, p1/z, [x1] +; VBITS_GE_256-NEXT: sel z0.s, p0, z0.s, z2.s +; VBITS_GE_256-NEXT: sel z1.s, p0, z1.s, z3.s +; VBITS_GE_256-NEXT: st1w { z0.s }, p1, [x0, x8, lsl #2] +; VBITS_GE_256-NEXT: st1w { z1.s }, p1, [x0] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: select_v16f32: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.s, vl16 ; VBITS_GE_512-NEXT: and w8, w2, #0x1 -; VBITS_GE_512-NEXT: ptrue p1.s +; VBITS_GE_512-NEXT: ptrue p0.s ; VBITS_GE_512-NEXT: mov z0.s, w8 -; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x0] -; VBITS_GE_512-NEXT: ld1w { z2.s }, p0/z, [x1] -; VBITS_GE_512-NEXT: cmpne p1.s, p1/z, z0.s, #0 -; VBITS_GE_512-NEXT: sel z0.s, p1, z1.s, z2.s -; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0] +; VBITS_GE_512-NEXT: ptrue p1.s, vl16 +; VBITS_GE_512-NEXT: cmpne p0.s, p0/z, z0.s, #0 +; VBITS_GE_512-NEXT: ld1w { z0.s }, p1/z, [x0] +; VBITS_GE_512-NEXT: ld1w { z1.s }, p1/z, [x1] +; VBITS_GE_512-NEXT: sel z0.s, p0, z0.s, z1.s +; VBITS_GE_512-NEXT: st1w { z0.s }, p1, [x0] ; VBITS_GE_512-NEXT: ret %op1 = load volatile <16 x float>, ptr %a %op2 = load volatile <16 x float>, ptr %b @@ -216,15 +216,15 @@ define void @select_v16f32(ptr %a, ptr %b, i1 %mask) #0 { define void @select_v32f32(ptr %a, ptr %b, i1 %mask) vscale_range(8,0) #0 { ; CHECK-LABEL: select_v32f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl32 ; CHECK-NEXT: and w8, w2, #0x1 -; CHECK-NEXT: ptrue p1.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z0.s, w8 -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.s, p1/z, z0.s, #0 -; CHECK-NEXT: sel z0.s, p1, z1.s, z2.s -; CHECK-NEXT: st1w { z0.s }, p0, [x0] +; CHECK-NEXT: ptrue p1.s, vl32 +; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0 +; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0] +; CHECK-NEXT: ld1w { z1.s }, p1/z, [x1] +; CHECK-NEXT: sel z0.s, p0, z0.s, z1.s +; CHECK-NEXT: st1w { z0.s }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <32 x float>, ptr %a %op2 = load volatile <32 x float>, ptr %b @@ -236,15 +236,15 @@ define void @select_v32f32(ptr %a, ptr %b, i1 %mask) vscale_range(8,0) #0 { define void @select_v64f32(ptr %a, ptr %b, i1 %mask) vscale_range(16,0) #0 { ; CHECK-LABEL: select_v64f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl64 ; CHECK-NEXT: and w8, w2, #0x1 -; CHECK-NEXT: ptrue p1.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z0.s, w8 -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.s, p1/z, z0.s, #0 -; CHECK-NEXT: sel z0.s, p1, z1.s, z2.s -; CHECK-NEXT: st1w { z0.s }, p0, [x0] +; CHECK-NEXT: ptrue p1.s, vl64 +; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0 +; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0] +; CHECK-NEXT: ld1w { z1.s }, p1/z, [x1] +; CHECK-NEXT: sel z0.s, p0, z0.s, z1.s +; CHECK-NEXT: st1w { z0.s }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <64 x float>, ptr %a %op2 = load volatile <64 x float>, ptr %b @@ -282,16 +282,16 @@ define <2 x double> @select_v2f64(<2 x double> %op1, <2 x double> %op2, i1 %mask define void @select_v4f64(ptr %a, ptr %b, i1 %mask) vscale_range(2,0) #0 { ; CHECK-LABEL: select_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: // kill: def $w2 killed $w2 def $x2 ; CHECK-NEXT: and x8, x2, #0x1 -; CHECK-NEXT: ptrue p1.d +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z0.d, x8 -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.d, p1/z, z0.d, #0 -; CHECK-NEXT: sel z0.d, p1, z1.d, z2.d -; CHECK-NEXT: st1d { z0.d }, p0, [x0] +; CHECK-NEXT: ptrue p1.d, vl4 +; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 +; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0] +; CHECK-NEXT: ld1d { z1.d }, p1/z, [x1] +; CHECK-NEXT: sel z0.d, p0, z0.d, z1.d +; CHECK-NEXT: st1d { z0.d }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <4 x double>, ptr %a %op2 = load volatile <4 x double>, ptr %b @@ -303,35 +303,35 @@ define void @select_v4f64(ptr %a, ptr %b, i1 %mask) vscale_range(2,0) #0 { define void @select_v8f64(ptr %a, ptr %b, i1 %mask) #0 { ; VBITS_GE_256-LABEL: select_v8f64: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: // kill: def $w2 killed $w2 def $x2 ; VBITS_GE_256-NEXT: and x8, x2, #0x1 -; VBITS_GE_256-NEXT: ptrue p1.d +; VBITS_GE_256-NEXT: ptrue p0.d ; VBITS_GE_256-NEXT: mov z0.d, x8 +; VBITS_GE_256-NEXT: ptrue p1.d, vl4 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 -; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0, x8, lsl #3] -; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x0] -; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x1, x8, lsl #3] -; VBITS_GE_256-NEXT: cmpne p1.d, p1/z, z0.d, #0 -; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1] -; VBITS_GE_256-NEXT: sel z1.d, p1, z1.d, z3.d -; VBITS_GE_256-NEXT: mov z0.d, p1/m, z2.d -; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0, x8, lsl #3] -; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0] +; VBITS_GE_256-NEXT: cmpne p0.d, p0/z, z0.d, #0 +; VBITS_GE_256-NEXT: ld1d { z0.d }, p1/z, [x0, x8, lsl #3] +; VBITS_GE_256-NEXT: ld1d { z1.d }, p1/z, [x0] +; VBITS_GE_256-NEXT: ld1d { z2.d }, p1/z, [x1, x8, lsl #3] +; VBITS_GE_256-NEXT: ld1d { z3.d }, p1/z, [x1] +; VBITS_GE_256-NEXT: sel z0.d, p0, z0.d, z2.d +; VBITS_GE_256-NEXT: sel z1.d, p0, z1.d, z3.d +; VBITS_GE_256-NEXT: st1d { z0.d }, p1, [x0, x8, lsl #3] +; VBITS_GE_256-NEXT: st1d { z1.d }, p1, [x0] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: select_v8f64: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.d, vl8 ; VBITS_GE_512-NEXT: // kill: def $w2 killed $w2 def $x2 ; VBITS_GE_512-NEXT: and x8, x2, #0x1 -; VBITS_GE_512-NEXT: ptrue p1.d +; VBITS_GE_512-NEXT: ptrue p0.d ; VBITS_GE_512-NEXT: mov z0.d, x8 -; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x0] -; VBITS_GE_512-NEXT: ld1d { z2.d }, p0/z, [x1] -; VBITS_GE_512-NEXT: cmpne p1.d, p1/z, z0.d, #0 -; VBITS_GE_512-NEXT: sel z0.d, p1, z1.d, z2.d -; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0] +; VBITS_GE_512-NEXT: ptrue p1.d, vl8 +; VBITS_GE_512-NEXT: cmpne p0.d, p0/z, z0.d, #0 +; VBITS_GE_512-NEXT: ld1d { z0.d }, p1/z, [x0] +; VBITS_GE_512-NEXT: ld1d { z1.d }, p1/z, [x1] +; VBITS_GE_512-NEXT: sel z0.d, p0, z0.d, z1.d +; VBITS_GE_512-NEXT: st1d { z0.d }, p1, [x0] ; VBITS_GE_512-NEXT: ret %op1 = load volatile <8 x double>, ptr %a %op2 = load volatile <8 x double>, ptr %b @@ -343,16 +343,16 @@ define void @select_v8f64(ptr %a, ptr %b, i1 %mask) #0 { define void @select_v16f64(ptr %a, ptr %b, i1 %mask) vscale_range(8,0) #0 { ; CHECK-LABEL: select_v16f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl16 ; CHECK-NEXT: // kill: def $w2 killed $w2 def $x2 ; CHECK-NEXT: and x8, x2, #0x1 -; CHECK-NEXT: ptrue p1.d +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z0.d, x8 -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.d, p1/z, z0.d, #0 -; CHECK-NEXT: sel z0.d, p1, z1.d, z2.d -; CHECK-NEXT: st1d { z0.d }, p0, [x0] +; CHECK-NEXT: ptrue p1.d, vl16 +; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 +; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0] +; CHECK-NEXT: ld1d { z1.d }, p1/z, [x1] +; CHECK-NEXT: sel z0.d, p0, z0.d, z1.d +; CHECK-NEXT: st1d { z0.d }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <16 x double>, ptr %a %op2 = load volatile <16 x double>, ptr %b @@ -364,16 +364,16 @@ define void @select_v16f64(ptr %a, ptr %b, i1 %mask) vscale_range(8,0) #0 { define void @select_v32f64(ptr %a, ptr %b, i1 %mask) vscale_range(16,0) #0 { ; CHECK-LABEL: select_v32f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl32 ; CHECK-NEXT: // kill: def $w2 killed $w2 def $x2 ; CHECK-NEXT: and x8, x2, #0x1 -; CHECK-NEXT: ptrue p1.d +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z0.d, x8 -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.d, p1/z, z0.d, #0 -; CHECK-NEXT: sel z0.d, p1, z1.d, z2.d -; CHECK-NEXT: st1d { z0.d }, p0, [x0] +; CHECK-NEXT: ptrue p1.d, vl32 +; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 +; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0] +; CHECK-NEXT: ld1d { z1.d }, p1/z, [x1] +; CHECK-NEXT: sel z0.d, p0, z0.d, z1.d +; CHECK-NEXT: st1d { z0.d }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <32 x double>, ptr %a %op2 = load volatile <32 x double>, ptr %b diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-to-int.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-to-int.ll index da0cf927d74d..af54b146c5b6 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-to-int.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-to-int.ll @@ -131,8 +131,8 @@ define <4 x i32> @fcvtzu_v4f16_v4i32(<4 x half> %op1) vscale_range(2,0) #0 { define void @fcvtzu_v8f16_v8i32(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: fcvtzu_v8f16_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: ldr q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: fcvtzu z0.s, p0/m, z0.h ; CHECK-NEXT: st1w { z0.s }, p0, [x1] @@ -357,7 +357,6 @@ define void @fcvtzu_v16f32_v16i16(ptr %a, ptr %b) #0 { ; VBITS_GE_256: // %bb.0: ; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: mov x8, #8 // =0x8 -; VBITS_GE_256-NEXT: ptrue p1.h, vl16 ; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2] ; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0] ; VBITS_GE_256-NEXT: fcvtzu z0.s, p0/m, z0.s @@ -366,7 +365,8 @@ define void @fcvtzu_v16f32_v16i16(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h ; VBITS_GE_256-NEXT: splice z1.h, p0, z1.h, z0.h -; VBITS_GE_256-NEXT: st1h { z1.h }, p1, [x1] +; VBITS_GE_256-NEXT: ptrue p0.h, vl16 +; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x1] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: fcvtzu_v16f32_v16i16: @@ -532,8 +532,8 @@ define <2 x i64> @fcvtzu_v2f32_v2i64(<2 x float> %op1) vscale_range(2,0) #0 { define void @fcvtzu_v4f32_v4i64(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: fcvtzu_v4f32_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: ldr q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: fcvtzu z0.d, p0/m, z0.s ; CHECK-NEXT: st1d { z0.d }, p0, [x1] @@ -752,7 +752,6 @@ define void @fcvtzu_v8f64_v8i32(ptr %a, ptr %b) #0 { ; VBITS_GE_256: // %bb.0: ; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 -; VBITS_GE_256-NEXT: ptrue p1.s, vl8 ; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] ; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0] ; VBITS_GE_256-NEXT: fcvtzu z0.d, p0/m, z0.d @@ -761,7 +760,8 @@ define void @fcvtzu_v8f64_v8i32(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s ; VBITS_GE_256-NEXT: splice z1.s, p0, z1.s, z0.s -; VBITS_GE_256-NEXT: st1w { z1.s }, p1, [x1] +; VBITS_GE_256-NEXT: ptrue p0.s, vl8 +; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x1] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: fcvtzu_v8f64_v8i32: @@ -1024,8 +1024,8 @@ define <4 x i32> @fcvtzs_v4f16_v4i32(<4 x half> %op1) vscale_range(2,0) #0 { define void @fcvtzs_v8f16_v8i32(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: fcvtzs_v8f16_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: ldr q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: fcvtzs z0.s, p0/m, z0.h ; CHECK-NEXT: st1w { z0.s }, p0, [x1] @@ -1250,7 +1250,6 @@ define void @fcvtzs_v16f32_v16i16(ptr %a, ptr %b) #0 { ; VBITS_GE_256: // %bb.0: ; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: mov x8, #8 // =0x8 -; VBITS_GE_256-NEXT: ptrue p1.h, vl16 ; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2] ; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0] ; VBITS_GE_256-NEXT: fcvtzs z0.s, p0/m, z0.s @@ -1259,7 +1258,8 @@ define void @fcvtzs_v16f32_v16i16(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h ; VBITS_GE_256-NEXT: splice z1.h, p0, z1.h, z0.h -; VBITS_GE_256-NEXT: st1h { z1.h }, p1, [x1] +; VBITS_GE_256-NEXT: ptrue p0.h, vl16 +; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x1] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: fcvtzs_v16f32_v16i16: @@ -1425,8 +1425,8 @@ define <2 x i64> @fcvtzs_v2f32_v2i64(<2 x float> %op1) vscale_range(2,0) #0 { define void @fcvtzs_v4f32_v4i64(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: fcvtzs_v4f32_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: ldr q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.s ; CHECK-NEXT: st1d { z0.d }, p0, [x1] @@ -1645,7 +1645,6 @@ define void @fcvtzs_v8f64_v8i32(ptr %a, ptr %b) #0 { ; VBITS_GE_256: // %bb.0: ; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 -; VBITS_GE_256-NEXT: ptrue p1.s, vl8 ; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] ; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0] ; VBITS_GE_256-NEXT: fcvtzs z0.d, p0/m, z0.d @@ -1654,7 +1653,8 @@ define void @fcvtzs_v8f64_v8i32(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s ; VBITS_GE_256-NEXT: splice z1.s, p0, z1.s, z0.s -; VBITS_GE_256-NEXT: st1w { z1.s }, p1, [x1] +; VBITS_GE_256-NEXT: ptrue p0.s, vl8 +; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x1] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: fcvtzs_v8f64_v8i32: diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp128.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp128.ll index 115f722986b5..61e04682fa0b 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp128.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp128.ll @@ -85,10 +85,10 @@ define void @fcvt_v4f128_v4f64(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-NEXT: str q0, [sp, #32] // 16-byte Folded Spill ; CHECK-NEXT: ldr q0, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: bl __trunctfdf2 -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldr q1, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: add x8, sp, #128 +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: mov v0.d[1], v1.d[0] ; CHECK-NEXT: ldr z1, [x8, #1, mul vl] // 16-byte Folded Reload ; CHECK-NEXT: splice z0.d, p0, z0.d, z1.d @@ -111,14 +111,14 @@ define void @fcvt_v4f128_v4f64(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-NEXT: ldr q0, [sp, #112] // 16-byte Folded Reload ; CHECK-NEXT: bl __trunctfdf2 ; CHECK-NEXT: ldr q1, [sp, #96] // 16-byte Folded Reload -; CHECK-NEXT: ptrue p1.d, vl2 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: add x8, sp, #128 -; CHECK-NEXT: ptrue p0.d, vl4 +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: mov v0.d[1], v1.d[0] ; CHECK-NEXT: ldr z1, [x8] // 16-byte Folded Reload ; CHECK-NEXT: mov x8, #4 // =0x4 -; CHECK-NEXT: splice z0.d, p1, z0.d, z1.d +; CHECK-NEXT: splice z0.d, p0, z0.d, z1.d +; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: st1d { z0.d }, p0, [x19, x8, lsl #3] ; CHECK-NEXT: add x8, sp, #128 ; CHECK-NEXT: ldr z0, [x8, #1, mul vl] // 16-byte Folded Reload diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-frame-offests-crash.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-frame-offests-crash.ll index a4b5ccd69fdb..1bd688d23050 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-frame-offests-crash.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-frame-offests-crash.ll @@ -21,24 +21,24 @@ define dso_local void @func1(ptr %v1, ptr %v2, ptr %v3, ptr %v4, ptr %v5, ptr %v ; CHECK-NEXT: .cfi_offset w22, -32 ; CHECK-NEXT: .cfi_offset w29, -48 ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: add x11, sp, #176 -; CHECK-NEXT: add x10, sp, #144 -; CHECK-NEXT: add x9, sp, #112 +; CHECK-NEXT: add x10, sp, #176 ; CHECK-NEXT: add x8, sp, #48 +; CHECK-NEXT: add x9, sp, #144 ; CHECK-NEXT: add x20, sp, #176 -; CHECK-NEXT: ldp x13, x12, [sp, #328] ; CHECK-NEXT: ldr x15, [sp, #104] +; CHECK-NEXT: ld1d { z3.d }, p0/z, [x10] +; CHECK-NEXT: ld1d { z0.d }, p0/z, [x8] +; CHECK-NEXT: add x8, sp, #112 +; CHECK-NEXT: ld1d { z2.d }, p0/z, [x9] +; CHECK-NEXT: ld1d { z1.d }, p0/z, [x8] ; CHECK-NEXT: ldur q4, [sp, #88] -; CHECK-NEXT: ldp x16, x17, [sp, #208] +; CHECK-NEXT: ldp x9, x8, [sp, #328] ; CHECK-NEXT: ldr x19, [sp, #272] +; CHECK-NEXT: ldp x11, x10, [sp, #312] +; CHECK-NEXT: ldp x13, x12, [sp, #296] +; CHECK-NEXT: ldp x18, x14, [sp, #280] +; CHECK-NEXT: ldp x16, x17, [sp, #208] ; CHECK-NEXT: ldp x21, x22, [sp, #352] -; CHECK-NEXT: ld1d { z3.d }, p0/z, [x11] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x10] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x9] -; CHECK-NEXT: ld1d { z0.d }, p0/z, [x8] -; CHECK-NEXT: ldp x8, x14, [sp, #312] -; CHECK-NEXT: ldp x10, x9, [sp, #296] -; CHECK-NEXT: ldp x18, x11, [sp, #280] ; CHECK-NEXT: st1d { z3.d }, p0, [x20] ; CHECK-NEXT: add x20, sp, #144 ; CHECK-NEXT: st1d { z2.d }, p0, [x20] @@ -53,10 +53,10 @@ define dso_local void @func1(ptr %v1, ptr %v2, ptr %v3, ptr %v4, ptr %v5, ptr %v ; CHECK-NEXT: stp x16, x17, [sp, #208] ; CHECK-NEXT: stur q4, [sp, #88] ; CHECK-NEXT: str x15, [sp, #104] -; CHECK-NEXT: stp x11, x10, [sp, #288] -; CHECK-NEXT: stp x9, x8, [sp, #304] -; CHECK-NEXT: stp x14, x13, [sp, #320] -; CHECK-NEXT: str x12, [sp, #336] +; CHECK-NEXT: stp x14, x13, [sp, #288] +; CHECK-NEXT: stp x12, x11, [sp, #304] +; CHECK-NEXT: stp x10, x9, [sp, #320] +; CHECK-NEXT: str x8, [sp, #336] ; CHECK-NEXT: ldr x29, [sp], #48 // 8-byte Folded Reload ; CHECK-NEXT: b func2 ptr %v9, ptr %v10, ptr %v11, ptr %v12, ptr %v13, ptr %v14, ptr %v15, ptr %v16, diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-insert-vector-elt.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-insert-vector-elt.ll index 977c528e2583..6f4d257039bc 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-insert-vector-elt.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-insert-vector-elt.ll @@ -36,16 +36,16 @@ define <8 x half> @insertelement_v8f16(<8 x half> %op1) vscale_range(2,0) #0 { define void @insertelement_v16f16(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: insertelement_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl16 ; CHECK-NEXT: mov w8, #15 // =0xf ; CHECK-NEXT: index z0.h, #0, #1 -; CHECK-NEXT: ptrue p1.h +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z1.h, w8 -; CHECK-NEXT: ld1h { z2.h }, p0/z, [x0] -; CHECK-NEXT: cmpeq p1.h, p1/z, z0.h, z1.h -; CHECK-NEXT: fmov h0, #5.00000000 -; CHECK-NEXT: mov z2.h, p1/m, h0 -; CHECK-NEXT: st1h { z2.h }, p0, [x1] +; CHECK-NEXT: ptrue p1.h, vl16 +; CHECK-NEXT: cmpeq p0.h, p0/z, z0.h, z1.h +; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0] +; CHECK-NEXT: fmov h1, #5.00000000 +; CHECK-NEXT: mov z0.h, p0/m, h1 +; CHECK-NEXT: st1h { z0.h }, p1, [x1] ; CHECK-NEXT: ret %op1 = load <16 x half>, ptr %a %r = insertelement <16 x half> %op1, half 5.0, i64 15 @@ -56,33 +56,33 @@ define void @insertelement_v16f16(ptr %a, ptr %b) vscale_range(2,0) #0 { define void @insertelement_v32f16(ptr %a, ptr %b) #0 { ; VBITS_GE_256-LABEL: insertelement_v32f16: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.h, vl16 ; VBITS_GE_256-NEXT: mov w8, #15 // =0xf ; VBITS_GE_256-NEXT: index z0.h, #0, #1 -; VBITS_GE_256-NEXT: ptrue p1.h +; VBITS_GE_256-NEXT: ptrue p0.h ; VBITS_GE_256-NEXT: mov z1.h, w8 +; VBITS_GE_256-NEXT: ptrue p1.h, vl16 ; VBITS_GE_256-NEXT: mov x8, #16 // =0x10 -; VBITS_GE_256-NEXT: fmov h2, #5.00000000 -; VBITS_GE_256-NEXT: ld1h { z3.h }, p0/z, [x0, x8, lsl #1] -; VBITS_GE_256-NEXT: cmpeq p1.h, p1/z, z0.h, z1.h -; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0] -; VBITS_GE_256-NEXT: mov z3.h, p1/m, h2 -; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x1] -; VBITS_GE_256-NEXT: st1h { z3.h }, p0, [x1, x8, lsl #1] +; VBITS_GE_256-NEXT: cmpeq p0.h, p0/z, z0.h, z1.h +; VBITS_GE_256-NEXT: fmov h0, #5.00000000 +; VBITS_GE_256-NEXT: ld1h { z1.h }, p1/z, [x0, x8, lsl #1] +; VBITS_GE_256-NEXT: mov z1.h, p0/m, h0 +; VBITS_GE_256-NEXT: ld1h { z0.h }, p1/z, [x0] +; VBITS_GE_256-NEXT: st1h { z1.h }, p1, [x1, x8, lsl #1] +; VBITS_GE_256-NEXT: st1h { z0.h }, p1, [x1] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: insertelement_v32f16: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.h, vl32 ; VBITS_GE_512-NEXT: mov w8, #31 // =0x1f ; VBITS_GE_512-NEXT: index z0.h, #0, #1 -; VBITS_GE_512-NEXT: ptrue p1.h +; VBITS_GE_512-NEXT: ptrue p0.h ; VBITS_GE_512-NEXT: mov z1.h, w8 -; VBITS_GE_512-NEXT: ld1h { z2.h }, p0/z, [x0] -; VBITS_GE_512-NEXT: cmpeq p1.h, p1/z, z0.h, z1.h -; VBITS_GE_512-NEXT: fmov h0, #5.00000000 -; VBITS_GE_512-NEXT: mov z2.h, p1/m, h0 -; VBITS_GE_512-NEXT: st1h { z2.h }, p0, [x1] +; VBITS_GE_512-NEXT: ptrue p1.h, vl32 +; VBITS_GE_512-NEXT: cmpeq p0.h, p0/z, z0.h, z1.h +; VBITS_GE_512-NEXT: ld1h { z0.h }, p1/z, [x0] +; VBITS_GE_512-NEXT: fmov h1, #5.00000000 +; VBITS_GE_512-NEXT: mov z0.h, p0/m, h1 +; VBITS_GE_512-NEXT: st1h { z0.h }, p1, [x1] ; VBITS_GE_512-NEXT: ret %op1 = load <32 x half>, ptr %a %r = insertelement <32 x half> %op1, half 5.0, i64 31 @@ -93,16 +93,16 @@ define void @insertelement_v32f16(ptr %a, ptr %b) #0 { define void @insertelement_v64f16(ptr %a, ptr %b) vscale_range(8,0) #0 { ; CHECK-LABEL: insertelement_v64f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl64 ; CHECK-NEXT: mov w8, #63 // =0x3f ; CHECK-NEXT: index z0.h, #0, #1 -; CHECK-NEXT: ptrue p1.h +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z1.h, w8 -; CHECK-NEXT: ld1h { z2.h }, p0/z, [x0] -; CHECK-NEXT: cmpeq p1.h, p1/z, z0.h, z1.h -; CHECK-NEXT: fmov h0, #5.00000000 -; CHECK-NEXT: mov z2.h, p1/m, h0 -; CHECK-NEXT: st1h { z2.h }, p0, [x1] +; CHECK-NEXT: ptrue p1.h, vl64 +; CHECK-NEXT: cmpeq p0.h, p0/z, z0.h, z1.h +; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0] +; CHECK-NEXT: fmov h1, #5.00000000 +; CHECK-NEXT: mov z0.h, p0/m, h1 +; CHECK-NEXT: st1h { z0.h }, p1, [x1] ; CHECK-NEXT: ret %op1 = load <64 x half>, ptr %a %r = insertelement <64 x half> %op1, half 5.0, i64 63 @@ -113,16 +113,16 @@ define void @insertelement_v64f16(ptr %a, ptr %b) vscale_range(8,0) #0 { define void @insertelement_v128f16(ptr %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-LABEL: insertelement_v128f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl128 ; CHECK-NEXT: mov w8, #127 // =0x7f ; CHECK-NEXT: index z0.h, #0, #1 -; CHECK-NEXT: ptrue p1.h +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z1.h, w8 -; CHECK-NEXT: ld1h { z2.h }, p0/z, [x0] -; CHECK-NEXT: cmpeq p1.h, p1/z, z0.h, z1.h -; CHECK-NEXT: fmov h0, #5.00000000 -; CHECK-NEXT: mov z2.h, p1/m, h0 -; CHECK-NEXT: st1h { z2.h }, p0, [x1] +; CHECK-NEXT: ptrue p1.h, vl128 +; CHECK-NEXT: cmpeq p0.h, p0/z, z0.h, z1.h +; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0] +; CHECK-NEXT: fmov h1, #5.00000000 +; CHECK-NEXT: mov z0.h, p0/m, h1 +; CHECK-NEXT: st1h { z0.h }, p1, [x1] ; CHECK-NEXT: ret %op1 = load <128 x half>, ptr %a %r = insertelement <128 x half> %op1, half 5.0, i64 127 @@ -157,16 +157,16 @@ define <4 x float> @insertelement_v4f32(<4 x float> %op1) vscale_range(2,0) #0 { define void @insertelement_v8f32(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: insertelement_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: mov w8, #7 // =0x7 ; CHECK-NEXT: index z0.s, #0, #1 -; CHECK-NEXT: ptrue p1.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x0] -; CHECK-NEXT: cmpeq p1.s, p1/z, z0.s, z1.s -; CHECK-NEXT: fmov s0, #5.00000000 -; CHECK-NEXT: mov z2.s, p1/m, s0 -; CHECK-NEXT: st1w { z2.s }, p0, [x1] +; CHECK-NEXT: ptrue p1.s, vl8 +; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, z1.s +; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0] +; CHECK-NEXT: fmov s1, #5.00000000 +; CHECK-NEXT: mov z0.s, p0/m, s1 +; CHECK-NEXT: st1w { z0.s }, p1, [x1] ; CHECK-NEXT: ret %op1 = load <8 x float>, ptr %a %r = insertelement <8 x float> %op1, float 5.0, i64 7 @@ -177,33 +177,33 @@ define void @insertelement_v8f32(ptr %a, ptr %b) vscale_range(2,0) #0 { define void @insertelement_v16f32(ptr %a, ptr %b) #0 { ; VBITS_GE_256-LABEL: insertelement_v16f32: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: mov w8, #7 // =0x7 ; VBITS_GE_256-NEXT: index z0.s, #0, #1 -; VBITS_GE_256-NEXT: ptrue p1.s +; VBITS_GE_256-NEXT: ptrue p0.s ; VBITS_GE_256-NEXT: mov z1.s, w8 +; VBITS_GE_256-NEXT: ptrue p1.s, vl8 ; VBITS_GE_256-NEXT: mov x8, #8 // =0x8 -; VBITS_GE_256-NEXT: fmov s2, #5.00000000 -; VBITS_GE_256-NEXT: ld1w { z3.s }, p0/z, [x0, x8, lsl #2] -; VBITS_GE_256-NEXT: cmpeq p1.s, p1/z, z0.s, z1.s -; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0] -; VBITS_GE_256-NEXT: mov z3.s, p1/m, s2 -; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x1] -; VBITS_GE_256-NEXT: st1w { z3.s }, p0, [x1, x8, lsl #2] +; VBITS_GE_256-NEXT: cmpeq p0.s, p0/z, z0.s, z1.s +; VBITS_GE_256-NEXT: fmov s0, #5.00000000 +; VBITS_GE_256-NEXT: ld1w { z1.s }, p1/z, [x0, x8, lsl #2] +; VBITS_GE_256-NEXT: mov z1.s, p0/m, s0 +; VBITS_GE_256-NEXT: ld1w { z0.s }, p1/z, [x0] +; VBITS_GE_256-NEXT: st1w { z1.s }, p1, [x1, x8, lsl #2] +; VBITS_GE_256-NEXT: st1w { z0.s }, p1, [x1] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: insertelement_v16f32: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.s, vl16 ; VBITS_GE_512-NEXT: mov w8, #15 // =0xf ; VBITS_GE_512-NEXT: index z0.s, #0, #1 -; VBITS_GE_512-NEXT: ptrue p1.s +; VBITS_GE_512-NEXT: ptrue p0.s ; VBITS_GE_512-NEXT: mov z1.s, w8 -; VBITS_GE_512-NEXT: ld1w { z2.s }, p0/z, [x0] -; VBITS_GE_512-NEXT: cmpeq p1.s, p1/z, z0.s, z1.s -; VBITS_GE_512-NEXT: fmov s0, #5.00000000 -; VBITS_GE_512-NEXT: mov z2.s, p1/m, s0 -; VBITS_GE_512-NEXT: st1w { z2.s }, p0, [x1] +; VBITS_GE_512-NEXT: ptrue p1.s, vl16 +; VBITS_GE_512-NEXT: cmpeq p0.s, p0/z, z0.s, z1.s +; VBITS_GE_512-NEXT: ld1w { z0.s }, p1/z, [x0] +; VBITS_GE_512-NEXT: fmov s1, #5.00000000 +; VBITS_GE_512-NEXT: mov z0.s, p0/m, s1 +; VBITS_GE_512-NEXT: st1w { z0.s }, p1, [x1] ; VBITS_GE_512-NEXT: ret %op1 = load <16 x float>, ptr %a %r = insertelement <16 x float> %op1, float 5.0, i64 15 @@ -214,16 +214,16 @@ define void @insertelement_v16f32(ptr %a, ptr %b) #0 { define void @insertelement_v32f32(ptr %a, ptr %b) vscale_range(8,0) #0 { ; CHECK-LABEL: insertelement_v32f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl32 ; CHECK-NEXT: mov w8, #31 // =0x1f ; CHECK-NEXT: index z0.s, #0, #1 -; CHECK-NEXT: ptrue p1.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x0] -; CHECK-NEXT: cmpeq p1.s, p1/z, z0.s, z1.s -; CHECK-NEXT: fmov s0, #5.00000000 -; CHECK-NEXT: mov z2.s, p1/m, s0 -; CHECK-NEXT: st1w { z2.s }, p0, [x1] +; CHECK-NEXT: ptrue p1.s, vl32 +; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, z1.s +; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0] +; CHECK-NEXT: fmov s1, #5.00000000 +; CHECK-NEXT: mov z0.s, p0/m, s1 +; CHECK-NEXT: st1w { z0.s }, p1, [x1] ; CHECK-NEXT: ret %op1 = load <32 x float>, ptr %a %r = insertelement <32 x float> %op1, float 5.0, i64 31 @@ -234,16 +234,16 @@ define void @insertelement_v32f32(ptr %a, ptr %b) vscale_range(8,0) #0 { define void @insertelement_v64f32(ptr %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-LABEL: insertelement_v64f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl64 ; CHECK-NEXT: mov w8, #63 // =0x3f ; CHECK-NEXT: index z0.s, #0, #1 -; CHECK-NEXT: ptrue p1.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x0] -; CHECK-NEXT: cmpeq p1.s, p1/z, z0.s, z1.s -; CHECK-NEXT: fmov s0, #5.00000000 -; CHECK-NEXT: mov z2.s, p1/m, s0 -; CHECK-NEXT: st1w { z2.s }, p0, [x1] +; CHECK-NEXT: ptrue p1.s, vl64 +; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, z1.s +; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0] +; CHECK-NEXT: fmov s1, #5.00000000 +; CHECK-NEXT: mov z0.s, p0/m, s1 +; CHECK-NEXT: st1w { z0.s }, p1, [x1] ; CHECK-NEXT: ret %op1 = load <64 x float>, ptr %a %r = insertelement <64 x float> %op1, float 5.0, i64 63 @@ -276,16 +276,16 @@ define <2 x double> @insertelement_v2f64(<2 x double> %op1) vscale_range(2,0) #0 define void @insertelement_v4f64(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: insertelement_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: mov w8, #3 // =0x3 ; CHECK-NEXT: index z0.d, #0, #1 -; CHECK-NEXT: ptrue p1.d +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0] -; CHECK-NEXT: cmpeq p1.d, p1/z, z0.d, z1.d -; CHECK-NEXT: fmov d0, #5.00000000 -; CHECK-NEXT: mov z2.d, p1/m, d0 -; CHECK-NEXT: st1d { z2.d }, p0, [x1] +; CHECK-NEXT: ptrue p1.d, vl4 +; CHECK-NEXT: cmpeq p0.d, p0/z, z0.d, z1.d +; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0] +; CHECK-NEXT: fmov d1, #5.00000000 +; CHECK-NEXT: mov z0.d, p0/m, d1 +; CHECK-NEXT: st1d { z0.d }, p1, [x1] ; CHECK-NEXT: ret %op1 = load <4 x double>, ptr %a %r = insertelement <4 x double> %op1, double 5.0, i64 3 @@ -296,33 +296,33 @@ define void @insertelement_v4f64(ptr %a, ptr %b) vscale_range(2,0) #0 { define void @insertelement_v8f64(ptr %a, ptr %b) #0 { ; VBITS_GE_256-LABEL: insertelement_v8f64: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: mov w8, #3 // =0x3 ; VBITS_GE_256-NEXT: index z0.d, #0, #1 -; VBITS_GE_256-NEXT: ptrue p1.d +; VBITS_GE_256-NEXT: ptrue p0.d ; VBITS_GE_256-NEXT: mov z1.d, x8 +; VBITS_GE_256-NEXT: ptrue p1.d, vl4 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 -; VBITS_GE_256-NEXT: fmov d2, #5.00000000 -; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x0, x8, lsl #3] -; VBITS_GE_256-NEXT: cmpeq p1.d, p1/z, z0.d, z1.d -; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0] -; VBITS_GE_256-NEXT: mov z3.d, p1/m, d2 -; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x1] -; VBITS_GE_256-NEXT: st1d { z3.d }, p0, [x1, x8, lsl #3] +; VBITS_GE_256-NEXT: cmpeq p0.d, p0/z, z0.d, z1.d +; VBITS_GE_256-NEXT: fmov d0, #5.00000000 +; VBITS_GE_256-NEXT: ld1d { z1.d }, p1/z, [x0, x8, lsl #3] +; VBITS_GE_256-NEXT: mov z1.d, p0/m, d0 +; VBITS_GE_256-NEXT: ld1d { z0.d }, p1/z, [x0] +; VBITS_GE_256-NEXT: st1d { z1.d }, p1, [x1, x8, lsl #3] +; VBITS_GE_256-NEXT: st1d { z0.d }, p1, [x1] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: insertelement_v8f64: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.d, vl8 ; VBITS_GE_512-NEXT: mov w8, #7 // =0x7 ; VBITS_GE_512-NEXT: index z0.d, #0, #1 -; VBITS_GE_512-NEXT: ptrue p1.d +; VBITS_GE_512-NEXT: ptrue p0.d ; VBITS_GE_512-NEXT: mov z1.d, x8 -; VBITS_GE_512-NEXT: ld1d { z2.d }, p0/z, [x0] -; VBITS_GE_512-NEXT: cmpeq p1.d, p1/z, z0.d, z1.d -; VBITS_GE_512-NEXT: fmov d0, #5.00000000 -; VBITS_GE_512-NEXT: mov z2.d, p1/m, d0 -; VBITS_GE_512-NEXT: st1d { z2.d }, p0, [x1] +; VBITS_GE_512-NEXT: ptrue p1.d, vl8 +; VBITS_GE_512-NEXT: cmpeq p0.d, p0/z, z0.d, z1.d +; VBITS_GE_512-NEXT: ld1d { z0.d }, p1/z, [x0] +; VBITS_GE_512-NEXT: fmov d1, #5.00000000 +; VBITS_GE_512-NEXT: mov z0.d, p0/m, d1 +; VBITS_GE_512-NEXT: st1d { z0.d }, p1, [x1] ; VBITS_GE_512-NEXT: ret %op1 = load <8 x double>, ptr %a %r = insertelement <8 x double> %op1, double 5.0, i64 7 @@ -333,16 +333,16 @@ define void @insertelement_v8f64(ptr %a, ptr %b) #0 { define void @insertelement_v16f64(ptr %a, ptr %b) vscale_range(8,0) #0 { ; CHECK-LABEL: insertelement_v16f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl16 ; CHECK-NEXT: mov w8, #15 // =0xf ; CHECK-NEXT: index z0.d, #0, #1 -; CHECK-NEXT: ptrue p1.d +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0] -; CHECK-NEXT: cmpeq p1.d, p1/z, z0.d, z1.d -; CHECK-NEXT: fmov d0, #5.00000000 -; CHECK-NEXT: mov z2.d, p1/m, d0 -; CHECK-NEXT: st1d { z2.d }, p0, [x1] +; CHECK-NEXT: ptrue p1.d, vl16 +; CHECK-NEXT: cmpeq p0.d, p0/z, z0.d, z1.d +; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0] +; CHECK-NEXT: fmov d1, #5.00000000 +; CHECK-NEXT: mov z0.d, p0/m, d1 +; CHECK-NEXT: st1d { z0.d }, p1, [x1] ; CHECK-NEXT: ret %op1 = load <16 x double>, ptr %a %r = insertelement <16 x double> %op1, double 5.0, i64 15 @@ -353,16 +353,16 @@ define void @insertelement_v16f64(ptr %a, ptr %b) vscale_range(8,0) #0 { define void @insertelement_v32f64(ptr %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-LABEL: insertelement_v32f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl32 ; CHECK-NEXT: mov w8, #31 // =0x1f ; CHECK-NEXT: index z0.d, #0, #1 -; CHECK-NEXT: ptrue p1.d +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0] -; CHECK-NEXT: cmpeq p1.d, p1/z, z0.d, z1.d -; CHECK-NEXT: fmov d0, #5.00000000 -; CHECK-NEXT: mov z2.d, p1/m, d0 -; CHECK-NEXT: st1d { z2.d }, p0, [x1] +; CHECK-NEXT: ptrue p1.d, vl32 +; CHECK-NEXT: cmpeq p0.d, p0/z, z0.d, z1.d +; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0] +; CHECK-NEXT: fmov d1, #5.00000000 +; CHECK-NEXT: mov z0.d, p0/m, d1 +; CHECK-NEXT: st1d { z0.d }, p1, [x1] ; CHECK-NEXT: ret %op1 = load <32 x double>, ptr %a %r = insertelement <32 x double> %op1, double 5.0, i64 31 diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-arith.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-arith.ll index b45a62f4e758..58fca3a2cf8b 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-arith.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-arith.ll @@ -1388,11 +1388,11 @@ define void @abs_v128i16(ptr %a) vscale_range(2,0) #0 { ; CHECK-NEXT: mov x10, #64 // =0x40 ; CHECK-NEXT: mov x11, #80 // =0x50 ; CHECK-NEXT: mov x12, #32 // =0x20 -; CHECK-NEXT: mov x13, #48 // =0x30 -; CHECK-NEXT: mov x14, #16 // =0x10 ; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1] ; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0, x9, lsl #1] ; CHECK-NEXT: ld1h { z2.h }, p0/z, [x0, x10, lsl #1] +; CHECK-NEXT: mov x13, #48 // =0x30 +; CHECK-NEXT: mov x14, #16 // =0x10 ; CHECK-NEXT: ld1h { z3.h }, p0/z, [x0, x11, lsl #1] ; CHECK-NEXT: ld1h { z4.h }, p0/z, [x0, x12, lsl #1] ; CHECK-NEXT: ld1h { z5.h }, p0/z, [x0, x13, lsl #1] diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-div.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-div.ll index 11ed69513917..0ddf434eff93 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-div.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-div.ll @@ -15,9 +15,9 @@ target triple = "aarch64-unknown-linux-gnu" define <8 x i8> @sdiv_v8i8(<8 x i8> %op1, <8 x i8> %op2) #0 { ; VBITS_GE_128-LABEL: sdiv_v8i8: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: sshll v1.8h, v1.8b, #0 ; VBITS_GE_128-NEXT: sshll v0.8h, v0.8b, #0 +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: sshll2 v2.4s, v1.8h, #0 ; VBITS_GE_128-NEXT: sshll2 v3.4s, v0.8h, #0 ; VBITS_GE_128-NEXT: sshll v1.4s, v1.4h, #0 @@ -94,11 +94,11 @@ define <8 x i8> @sdiv_v8i8(<8 x i8> %op1, <8 x i8> %op2) #0 { define <16 x i8> @sdiv_v16i8(<16 x i8> %op1, <16 x i8> %op2) #0 { ; VBITS_GE_128-LABEL: sdiv_v16i8: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: sshll2 v2.8h, v1.16b, #0 ; VBITS_GE_128-NEXT: sshll2 v3.8h, v0.16b, #0 ; VBITS_GE_128-NEXT: sshll v1.8h, v1.8b, #0 ; VBITS_GE_128-NEXT: sshll v0.8h, v0.8b, #0 +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: sshll2 v4.4s, v2.8h, #0 ; VBITS_GE_128-NEXT: sshll2 v5.4s, v3.8h, #0 ; VBITS_GE_128-NEXT: sshll v2.4s, v2.4h, #0 @@ -203,7 +203,6 @@ define void @sdiv_v128i8(ptr %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-LABEL: sdiv_v128i8: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.b, vl128 -; CHECK-NEXT: ptrue p1.h, vl128 ; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0] ; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1] ; CHECK-NEXT: ptrue p0.s, vl64 @@ -221,7 +220,8 @@ define void @sdiv_v128i8(ptr %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-NEXT: uzp1 z1.h, z2.h, z2.h ; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: splice z1.h, p0, z1.h, z0.h -; CHECK-NEXT: st1b { z1.h }, p1, [x0] +; CHECK-NEXT: ptrue p0.h, vl128 +; CHECK-NEXT: st1b { z1.h }, p0, [x0] ; CHECK-NEXT: ret %op1 = load <128 x i8>, ptr %a %op2 = load <128 x i8>, ptr %b @@ -260,14 +260,14 @@ define void @sdiv_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-NEXT: sdivr z3.s, p1/m, z3.s, z5.s ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h ; CHECK-NEXT: sdiv z0.s, p1/m, z0.s, z1.s -; CHECK-NEXT: ptrue p1.h, vl64 ; CHECK-NEXT: uzp1 z1.h, z4.h, z4.h +; CHECK-NEXT: ptrue p1.h, vl64 ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h ; CHECK-NEXT: splice z1.h, p1, z1.h, z2.h ; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: splice z3.h, p1, z3.h, z0.h -; CHECK-NEXT: ptrue p1.b, vl128 ; CHECK-NEXT: uzp1 z0.b, z1.b, z1.b +; CHECK-NEXT: ptrue p1.b, vl128 ; CHECK-NEXT: uzp1 z1.b, z3.b, z3.b ; CHECK-NEXT: splice z0.b, p1, z0.b, z1.b ; CHECK-NEXT: st1b { z0.b }, p0, [x0] @@ -284,18 +284,18 @@ define void @sdiv_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 { define <4 x i16> @sdiv_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 { ; VBITS_GE_128-LABEL: sdiv_v4i16: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: sshll v1.4s, v1.4h, #0 ; VBITS_GE_128-NEXT: sshll v0.4s, v0.4h, #0 +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: sdiv z0.s, p0/m, z0.s, z1.s ; VBITS_GE_128-NEXT: xtn v0.4h, v0.4s ; VBITS_GE_128-NEXT: ret ; ; VBITS_GE_256-LABEL: sdiv_v4i16: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.s, vl4 ; VBITS_GE_256-NEXT: sshll v1.4s, v1.4h, #0 ; VBITS_GE_256-NEXT: sshll v0.4s, v0.4h, #0 +; VBITS_GE_256-NEXT: ptrue p0.s, vl4 ; VBITS_GE_256-NEXT: sdivr z1.s, p0/m, z1.s, z0.s ; VBITS_GE_256-NEXT: mov w8, v1.s[1] ; VBITS_GE_256-NEXT: mov v0.16b, v1.16b @@ -309,9 +309,9 @@ define <4 x i16> @sdiv_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 { ; ; VBITS_GE_512-LABEL: sdiv_v4i16: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.s, vl4 ; VBITS_GE_512-NEXT: sshll v1.4s, v1.4h, #0 ; VBITS_GE_512-NEXT: sshll v0.4s, v0.4h, #0 +; VBITS_GE_512-NEXT: ptrue p0.s, vl4 ; VBITS_GE_512-NEXT: sdivr z1.s, p0/m, z1.s, z0.s ; VBITS_GE_512-NEXT: mov w8, v1.s[1] ; VBITS_GE_512-NEXT: mov v0.16b, v1.16b @@ -329,11 +329,11 @@ define <4 x i16> @sdiv_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 { define <8 x i16> @sdiv_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 { ; VBITS_GE_128-LABEL: sdiv_v8i16: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: sshll2 v2.4s, v1.8h, #0 ; VBITS_GE_128-NEXT: sshll2 v3.4s, v0.8h, #0 ; VBITS_GE_128-NEXT: sshll v1.4s, v1.4h, #0 ; VBITS_GE_128-NEXT: sshll v0.4s, v0.4h, #0 +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: sdivr z2.s, p0/m, z2.s, z3.s ; VBITS_GE_128-NEXT: sdiv z0.s, p0/m, z0.s, z1.s ; VBITS_GE_128-NEXT: uzp1 v0.8h, v0.8h, v2.8h @@ -341,9 +341,9 @@ define <8 x i16> @sdiv_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 { ; ; VBITS_GE_256-LABEL: sdiv_v8i16: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: // kill: def $q1 killed $q1 def $z1 ; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 def $z0 +; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h ; VBITS_GE_256-NEXT: sunpklo z0.s, z0.h ; VBITS_GE_256-NEXT: sdiv z0.s, p0/m, z0.s, z1.s @@ -353,9 +353,9 @@ define <8 x i16> @sdiv_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 { ; ; VBITS_GE_512-LABEL: sdiv_v8i16: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.s, vl8 ; VBITS_GE_512-NEXT: // kill: def $q1 killed $q1 def $z1 ; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 def $z0 +; VBITS_GE_512-NEXT: ptrue p0.s, vl8 ; VBITS_GE_512-NEXT: sunpklo z1.s, z1.h ; VBITS_GE_512-NEXT: sunpklo z0.s, z0.h ; VBITS_GE_512-NEXT: sdiv z0.s, p0/m, z0.s, z1.s @@ -369,8 +369,8 @@ define <8 x i16> @sdiv_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 { define void @sdiv_v16i16(ptr %a, ptr %b) #0 { ; VBITS_GE_128-LABEL: sdiv_v16i16: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ldp q4, q1, [x1] +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ldr q0, [x0, #16] ; VBITS_GE_128-NEXT: sshll2 v2.4s, v1.8h, #0 ; VBITS_GE_128-NEXT: sshll2 v3.4s, v0.8h, #0 @@ -542,8 +542,8 @@ define void @sdiv_v8i32(ptr %a, ptr %b) vscale_range(2,0) #0 { define void @sdiv_v16i32(ptr %a, ptr %b) #0 { ; VBITS_GE_128-LABEL: sdiv_v16i32: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ldp q0, q3, [x1] +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ldp q1, q2, [x0] ; VBITS_GE_128-NEXT: ldp q5, q4, [x1, #32] ; VBITS_GE_128-NEXT: sdivr z0.s, p0/m, z0.s, z1.s @@ -664,8 +664,8 @@ define void @sdiv_v4i64(ptr %a, ptr %b) vscale_range(2,0) #0 { define void @sdiv_v8i64(ptr %a, ptr %b) #0 { ; VBITS_GE_128-LABEL: sdiv_v8i64: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.d, vl2 ; VBITS_GE_128-NEXT: ldp q0, q3, [x1] +; VBITS_GE_128-NEXT: ptrue p0.d, vl2 ; VBITS_GE_128-NEXT: ldp q1, q2, [x0] ; VBITS_GE_128-NEXT: ldp q5, q4, [x1, #32] ; VBITS_GE_128-NEXT: sdivr z0.d, p0/m, z0.d, z1.d @@ -748,9 +748,9 @@ define void @sdiv_v32i64(ptr %a, ptr %b) vscale_range(16,0) #0 { define <8 x i8> @udiv_v8i8(<8 x i8> %op1, <8 x i8> %op2) #0 { ; VBITS_GE_128-LABEL: udiv_v8i8: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ushll v1.8h, v1.8b, #0 ; VBITS_GE_128-NEXT: ushll v0.8h, v0.8b, #0 +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ushll2 v2.4s, v1.8h, #0 ; VBITS_GE_128-NEXT: ushll2 v3.4s, v0.8h, #0 ; VBITS_GE_128-NEXT: ushll v1.4s, v1.4h, #0 @@ -827,11 +827,11 @@ define <8 x i8> @udiv_v8i8(<8 x i8> %op1, <8 x i8> %op2) #0 { define <16 x i8> @udiv_v16i8(<16 x i8> %op1, <16 x i8> %op2) #0 { ; VBITS_GE_128-LABEL: udiv_v16i8: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ushll2 v2.8h, v1.16b, #0 ; VBITS_GE_128-NEXT: ushll2 v3.8h, v0.16b, #0 ; VBITS_GE_128-NEXT: ushll v1.8h, v1.8b, #0 ; VBITS_GE_128-NEXT: ushll v0.8h, v0.8b, #0 +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ushll2 v4.4s, v2.8h, #0 ; VBITS_GE_128-NEXT: ushll2 v5.4s, v3.8h, #0 ; VBITS_GE_128-NEXT: ushll v2.4s, v2.4h, #0 @@ -980,14 +980,14 @@ define void @udiv_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-NEXT: udivr z3.s, p1/m, z3.s, z5.s ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h ; CHECK-NEXT: udiv z0.s, p1/m, z0.s, z1.s -; CHECK-NEXT: ptrue p1.h, vl64 ; CHECK-NEXT: uzp1 z1.h, z4.h, z4.h +; CHECK-NEXT: ptrue p1.h, vl64 ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h ; CHECK-NEXT: splice z1.h, p1, z1.h, z2.h ; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: splice z3.h, p1, z3.h, z0.h -; CHECK-NEXT: ptrue p1.b, vl128 ; CHECK-NEXT: uzp1 z0.b, z1.b, z1.b +; CHECK-NEXT: ptrue p1.b, vl128 ; CHECK-NEXT: uzp1 z1.b, z3.b, z3.b ; CHECK-NEXT: splice z0.b, p1, z0.b, z1.b ; CHECK-NEXT: st1b { z0.b }, p0, [x0] @@ -1004,18 +1004,18 @@ define void @udiv_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 { define <4 x i16> @udiv_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 { ; VBITS_GE_128-LABEL: udiv_v4i16: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ushll v1.4s, v1.4h, #0 ; VBITS_GE_128-NEXT: ushll v0.4s, v0.4h, #0 +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: udiv z0.s, p0/m, z0.s, z1.s ; VBITS_GE_128-NEXT: xtn v0.4h, v0.4s ; VBITS_GE_128-NEXT: ret ; ; VBITS_GE_256-LABEL: udiv_v4i16: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.s, vl4 ; VBITS_GE_256-NEXT: ushll v1.4s, v1.4h, #0 ; VBITS_GE_256-NEXT: ushll v0.4s, v0.4h, #0 +; VBITS_GE_256-NEXT: ptrue p0.s, vl4 ; VBITS_GE_256-NEXT: udivr z1.s, p0/m, z1.s, z0.s ; VBITS_GE_256-NEXT: mov w8, v1.s[1] ; VBITS_GE_256-NEXT: mov v0.16b, v1.16b @@ -1029,9 +1029,9 @@ define <4 x i16> @udiv_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 { ; ; VBITS_GE_512-LABEL: udiv_v4i16: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.s, vl4 ; VBITS_GE_512-NEXT: ushll v1.4s, v1.4h, #0 ; VBITS_GE_512-NEXT: ushll v0.4s, v0.4h, #0 +; VBITS_GE_512-NEXT: ptrue p0.s, vl4 ; VBITS_GE_512-NEXT: udivr z1.s, p0/m, z1.s, z0.s ; VBITS_GE_512-NEXT: mov w8, v1.s[1] ; VBITS_GE_512-NEXT: mov v0.16b, v1.16b @@ -1049,11 +1049,11 @@ define <4 x i16> @udiv_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 { define <8 x i16> @udiv_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 { ; VBITS_GE_128-LABEL: udiv_v8i16: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ushll2 v2.4s, v1.8h, #0 ; VBITS_GE_128-NEXT: ushll2 v3.4s, v0.8h, #0 ; VBITS_GE_128-NEXT: ushll v1.4s, v1.4h, #0 ; VBITS_GE_128-NEXT: ushll v0.4s, v0.4h, #0 +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: udivr z2.s, p0/m, z2.s, z3.s ; VBITS_GE_128-NEXT: udiv z0.s, p0/m, z0.s, z1.s ; VBITS_GE_128-NEXT: uzp1 v0.8h, v0.8h, v2.8h @@ -1061,9 +1061,9 @@ define <8 x i16> @udiv_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 { ; ; VBITS_GE_256-LABEL: udiv_v8i16: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: // kill: def $q1 killed $q1 def $z1 ; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 def $z0 +; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: uunpklo z1.s, z1.h ; VBITS_GE_256-NEXT: uunpklo z0.s, z0.h ; VBITS_GE_256-NEXT: udiv z0.s, p0/m, z0.s, z1.s @@ -1073,9 +1073,9 @@ define <8 x i16> @udiv_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 { ; ; VBITS_GE_512-LABEL: udiv_v8i16: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.s, vl8 ; VBITS_GE_512-NEXT: // kill: def $q1 killed $q1 def $z1 ; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 def $z0 +; VBITS_GE_512-NEXT: ptrue p0.s, vl8 ; VBITS_GE_512-NEXT: uunpklo z1.s, z1.h ; VBITS_GE_512-NEXT: uunpklo z0.s, z0.h ; VBITS_GE_512-NEXT: udiv z0.s, p0/m, z0.s, z1.s @@ -1089,8 +1089,8 @@ define <8 x i16> @udiv_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 { define void @udiv_v16i16(ptr %a, ptr %b) #0 { ; VBITS_GE_128-LABEL: udiv_v16i16: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ldp q4, q1, [x1] +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ldr q0, [x0, #16] ; VBITS_GE_128-NEXT: ushll2 v2.4s, v1.8h, #0 ; VBITS_GE_128-NEXT: ushll2 v3.4s, v0.8h, #0 @@ -1253,8 +1253,8 @@ define void @udiv_v8i32(ptr %a, ptr %b) vscale_range(2,0) #0 { define void @udiv_v16i32(ptr %a, ptr %b) #0 { ; VBITS_GE_128-LABEL: udiv_v16i32: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ldp q0, q3, [x1] +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ldp q1, q2, [x0] ; VBITS_GE_128-NEXT: ldp q5, q4, [x1, #32] ; VBITS_GE_128-NEXT: udivr z0.s, p0/m, z0.s, z1.s @@ -1375,8 +1375,8 @@ define void @udiv_v4i64(ptr %a, ptr %b) vscale_range(2,0) #0 { define void @udiv_v8i64(ptr %a, ptr %b) #0 { ; VBITS_GE_128-LABEL: udiv_v8i64: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.d, vl2 ; VBITS_GE_128-NEXT: ldp q0, q3, [x1] +; VBITS_GE_128-NEXT: ptrue p0.d, vl2 ; VBITS_GE_128-NEXT: ldp q1, q2, [x0] ; VBITS_GE_128-NEXT: ldp q5, q4, [x1, #32] ; VBITS_GE_128-NEXT: udivr z0.d, p0/m, z0.d, z1.d diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-extends.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-extends.ll index 756e5f4cddf8..4feb86305f8f 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-extends.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-extends.ll @@ -58,8 +58,8 @@ define void @sext_v4i3_v4i64(<4 x i3> %a, ptr %out) vscale_range(2,0) #0 { define void @sext_v16i8_v16i16(<16 x i8> %a, ptr %out) vscale_range(2,0) #0 { ; CHECK-LABEL: sext_v16i8_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl16 ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: ptrue p0.h, vl16 ; CHECK-NEXT: sunpklo z0.h, z0.b ; CHECK-NEXT: st1h { z0.h }, p0, [x0] ; CHECK-NEXT: ret @@ -308,8 +308,8 @@ define void @sext_v32i8_v32i64(ptr %in, ptr %out) vscale_range(16,0) #0 { define void @sext_v8i16_v8i32(<8 x i16> %a, ptr %out) vscale_range(2,0) #0 { ; CHECK-LABEL: sext_v8i16_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: sunpklo z0.s, z0.h ; CHECK-NEXT: st1w { z0.s }, p0, [x0] ; CHECK-NEXT: ret @@ -472,8 +472,8 @@ define void @sext_v32i16_v32i64(ptr %in, ptr %out) vscale_range(16,0) #0 { define void @sext_v4i32_v4i64(<4 x i32> %a, ptr %out) vscale_range(2,0) #0 { ; CHECK-LABEL: sext_v4i32_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: sunpklo z0.d, z0.s ; CHECK-NEXT: st1d { z0.d }, p0, [x0] ; CHECK-NEXT: ret @@ -554,8 +554,8 @@ define void @sext_v32i32_v32i64(ptr %in, ptr %out) vscale_range(16,0) #0 { define void @zext_v16i8_v16i16(<16 x i8> %a, ptr %out) vscale_range(2,0) #0 { ; CHECK-LABEL: zext_v16i8_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl16 ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: ptrue p0.h, vl16 ; CHECK-NEXT: uunpklo z0.h, z0.b ; CHECK-NEXT: st1h { z0.h }, p0, [x0] ; CHECK-NEXT: ret @@ -804,8 +804,8 @@ define void @zext_v32i8_v32i64(ptr %in, ptr %out) vscale_range(16,0) #0 { define void @zext_v8i16_v8i32(<8 x i16> %a, ptr %out) vscale_range(2,0) #0 { ; CHECK-LABEL: zext_v8i16_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: st1w { z0.s }, p0, [x0] ; CHECK-NEXT: ret @@ -968,8 +968,8 @@ define void @zext_v32i16_v32i64(ptr %in, ptr %out) vscale_range(16,0) #0 { define void @zext_v4i32_v4i64(<4 x i32> %a, ptr %out) vscale_range(2,0) #0 { ; CHECK-LABEL: zext_v4i32_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: st1d { z0.d }, p0, [x0] ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-rem.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-rem.ll index 38444d83c1d7..2d7894539917 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-rem.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-rem.ll @@ -15,9 +15,9 @@ target triple = "aarch64-unknown-linux-gnu" define <8 x i8> @srem_v8i8(<8 x i8> %op1, <8 x i8> %op2) #0 { ; VBITS_GE_128-LABEL: srem_v8i8: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: sshll v2.8h, v1.8b, #0 ; VBITS_GE_128-NEXT: sshll v3.8h, v0.8b, #0 +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: sshll2 v4.4s, v2.8h, #0 ; VBITS_GE_128-NEXT: sshll2 v5.4s, v3.8h, #0 ; VBITS_GE_128-NEXT: sshll v2.4s, v2.4h, #0 @@ -97,9 +97,9 @@ define <8 x i8> @srem_v8i8(<8 x i8> %op1, <8 x i8> %op2) #0 { define <16 x i8> @srem_v16i8(<16 x i8> %op1, <16 x i8> %op2) #0 { ; VBITS_GE_128-LABEL: srem_v16i8: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: sshll2 v2.8h, v1.16b, #0 ; VBITS_GE_128-NEXT: sshll2 v3.8h, v0.16b, #0 +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: sshll2 v4.4s, v2.8h, #0 ; VBITS_GE_128-NEXT: sshll2 v5.4s, v3.8h, #0 ; VBITS_GE_128-NEXT: sshll v2.4s, v2.4h, #0 @@ -277,8 +277,8 @@ define void @srem_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h ; CHECK-NEXT: sdivr z3.s, p1/m, z3.s, z5.s ; CHECK-NEXT: ptrue p1.h, vl64 -; CHECK-NEXT: uzp1 z5.h, z6.h, z6.h ; CHECK-NEXT: splice z4.h, p1, z4.h, z2.h +; CHECK-NEXT: uzp1 z5.h, z6.h, z6.h ; CHECK-NEXT: uzp1 z2.b, z4.b, z4.b ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h ; CHECK-NEXT: splice z5.h, p1, z5.h, z3.h @@ -300,9 +300,9 @@ define void @srem_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 { define <4 x i16> @srem_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 { ; VBITS_GE_128-LABEL: srem_v4i16: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: sshll v2.4s, v1.4h, #0 ; VBITS_GE_128-NEXT: sshll v3.4s, v0.4h, #0 +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: sdivr z2.s, p0/m, z2.s, z3.s ; VBITS_GE_128-NEXT: xtn v2.4h, v2.4s ; VBITS_GE_128-NEXT: mls v0.4h, v2.4h, v1.4h @@ -310,9 +310,9 @@ define <4 x i16> @srem_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 { ; ; VBITS_GE_256-LABEL: srem_v4i16: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.s, vl4 ; VBITS_GE_256-NEXT: sshll v2.4s, v1.4h, #0 ; VBITS_GE_256-NEXT: sshll v3.4s, v0.4h, #0 +; VBITS_GE_256-NEXT: ptrue p0.s, vl4 ; VBITS_GE_256-NEXT: sdivr z2.s, p0/m, z2.s, z3.s ; VBITS_GE_256-NEXT: mov w8, v2.s[1] ; VBITS_GE_256-NEXT: mov v3.16b, v2.16b @@ -326,9 +326,9 @@ define <4 x i16> @srem_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 { ; ; VBITS_GE_512-LABEL: srem_v4i16: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.s, vl4 ; VBITS_GE_512-NEXT: sshll v2.4s, v1.4h, #0 ; VBITS_GE_512-NEXT: sshll v3.4s, v0.4h, #0 +; VBITS_GE_512-NEXT: ptrue p0.s, vl4 ; VBITS_GE_512-NEXT: sdivr z2.s, p0/m, z2.s, z3.s ; VBITS_GE_512-NEXT: mov w8, v2.s[1] ; VBITS_GE_512-NEXT: mov v3.16b, v2.16b @@ -346,9 +346,9 @@ define <4 x i16> @srem_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 { define <8 x i16> @srem_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 { ; VBITS_GE_128-LABEL: srem_v8i16: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: sshll2 v2.4s, v1.8h, #0 ; VBITS_GE_128-NEXT: sshll2 v3.4s, v0.8h, #0 +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: sshll v4.4s, v0.4h, #0 ; VBITS_GE_128-NEXT: sdivr z2.s, p0/m, z2.s, z3.s ; VBITS_GE_128-NEXT: sshll v3.4s, v1.4h, #0 @@ -359,11 +359,11 @@ define <8 x i16> @srem_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 { ; ; VBITS_GE_256-LABEL: srem_v8i16: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: // kill: def $q1 killed $q1 def $z1 ; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 def $z0 ; VBITS_GE_256-NEXT: sunpklo z2.s, z1.h ; VBITS_GE_256-NEXT: sunpklo z3.s, z0.h +; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: sdivr z2.s, p0/m, z2.s, z3.s ; VBITS_GE_256-NEXT: uzp1 z2.h, z2.h, z2.h ; VBITS_GE_256-NEXT: mls v0.8h, v2.8h, v1.8h @@ -372,11 +372,11 @@ define <8 x i16> @srem_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 { ; ; VBITS_GE_512-LABEL: srem_v8i16: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.s, vl8 ; VBITS_GE_512-NEXT: // kill: def $q1 killed $q1 def $z1 ; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 def $z0 ; VBITS_GE_512-NEXT: sunpklo z2.s, z1.h ; VBITS_GE_512-NEXT: sunpklo z3.s, z0.h +; VBITS_GE_512-NEXT: ptrue p0.s, vl8 ; VBITS_GE_512-NEXT: sdivr z2.s, p0/m, z2.s, z3.s ; VBITS_GE_512-NEXT: uzp1 z2.h, z2.h, z2.h ; VBITS_GE_512-NEXT: mls v0.8h, v2.8h, v1.8h @@ -389,8 +389,8 @@ define <8 x i16> @srem_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 { define void @srem_v16i16(ptr %a, ptr %b) #0 { ; VBITS_GE_128-LABEL: srem_v16i16: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ldp q4, q1, [x1] +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ldr q0, [x0, #16] ; VBITS_GE_128-NEXT: sshll2 v2.4s, v1.8h, #0 ; VBITS_GE_128-NEXT: sshll2 v3.4s, v0.8h, #0 @@ -582,25 +582,25 @@ define void @srem_v8i32(ptr %a, ptr %b) vscale_range(2,0) #0 { define void @srem_v16i32(ptr %a, ptr %b) #0 { ; VBITS_GE_128-LABEL: srem_v16i32: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ldp q0, q3, [x1] +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ldp q1, q2, [x0] ; VBITS_GE_128-NEXT: ldp q16, q5, [x0, #32] ; VBITS_GE_128-NEXT: ldp q17, q6, [x1, #32] ; VBITS_GE_128-NEXT: movprfx z4, z1 ; VBITS_GE_128-NEXT: sdiv z4.s, p0/m, z4.s, z0.s +; VBITS_GE_128-NEXT: movprfx z19, z2 +; VBITS_GE_128-NEXT: sdiv z19.s, p0/m, z19.s, z3.s ; VBITS_GE_128-NEXT: movprfx z7, z5 ; VBITS_GE_128-NEXT: sdiv z7.s, p0/m, z7.s, z6.s ; VBITS_GE_128-NEXT: movprfx z18, z16 ; VBITS_GE_128-NEXT: sdiv z18.s, p0/m, z18.s, z17.s -; VBITS_GE_128-NEXT: movprfx z19, z2 -; VBITS_GE_128-NEXT: sdiv z19.s, p0/m, z19.s, z3.s -; VBITS_GE_128-NEXT: mls v16.4s, v18.4s, v17.4s -; VBITS_GE_128-NEXT: mls v5.4s, v7.4s, v6.4s ; VBITS_GE_128-NEXT: mls v1.4s, v4.4s, v0.4s ; VBITS_GE_128-NEXT: mls v2.4s, v19.4s, v3.4s -; VBITS_GE_128-NEXT: stp q16, q5, [x0, #32] +; VBITS_GE_128-NEXT: mls v16.4s, v18.4s, v17.4s +; VBITS_GE_128-NEXT: mls v5.4s, v7.4s, v6.4s ; VBITS_GE_128-NEXT: stp q1, q2, [x0] +; VBITS_GE_128-NEXT: stp q16, q5, [x0, #32] ; VBITS_GE_128-NEXT: ret ; ; VBITS_GE_256-LABEL: srem_v16i32: @@ -730,26 +730,26 @@ define void @srem_v4i64(ptr %a, ptr %b) vscale_range(2,0) #0 { define void @srem_v8i64(ptr %a, ptr %b) #0 { ; VBITS_GE_128-LABEL: srem_v8i64: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.d, vl2 ; VBITS_GE_128-NEXT: ldp q0, q3, [x1] +; VBITS_GE_128-NEXT: ptrue p0.d, vl2 ; VBITS_GE_128-NEXT: ldp q1, q2, [x0] ; VBITS_GE_128-NEXT: ldp q16, q5, [x0, #32] ; VBITS_GE_128-NEXT: ldp q17, q6, [x1, #32] ; VBITS_GE_128-NEXT: movprfx z4, z1 ; VBITS_GE_128-NEXT: sdiv z4.d, p0/m, z4.d, z0.d +; VBITS_GE_128-NEXT: movprfx z19, z2 +; VBITS_GE_128-NEXT: sdiv z19.d, p0/m, z19.d, z3.d ; VBITS_GE_128-NEXT: movprfx z7, z5 ; VBITS_GE_128-NEXT: sdiv z7.d, p0/m, z7.d, z6.d ; VBITS_GE_128-NEXT: movprfx z18, z16 ; VBITS_GE_128-NEXT: sdiv z18.d, p0/m, z18.d, z17.d -; VBITS_GE_128-NEXT: movprfx z19, z2 -; VBITS_GE_128-NEXT: sdiv z19.d, p0/m, z19.d, z3.d -; VBITS_GE_128-NEXT: mls z16.d, p0/m, z18.d, z17.d -; VBITS_GE_128-NEXT: mls z5.d, p0/m, z7.d, z6.d ; VBITS_GE_128-NEXT: msb z0.d, p0/m, z4.d, z1.d ; VBITS_GE_128-NEXT: movprfx z1, z2 ; VBITS_GE_128-NEXT: mls z1.d, p0/m, z19.d, z3.d -; VBITS_GE_128-NEXT: stp q16, q5, [x0, #32] +; VBITS_GE_128-NEXT: mls z16.d, p0/m, z18.d, z17.d +; VBITS_GE_128-NEXT: mls z5.d, p0/m, z7.d, z6.d ; VBITS_GE_128-NEXT: stp q0, q1, [x0] +; VBITS_GE_128-NEXT: stp q16, q5, [x0, #32] ; VBITS_GE_128-NEXT: ret ; ; VBITS_GE_256-LABEL: srem_v8i64: @@ -833,9 +833,9 @@ define void @srem_v32i64(ptr %a, ptr %b) vscale_range(16,0) #0 { define <8 x i8> @urem_v8i8(<8 x i8> %op1, <8 x i8> %op2) #0 { ; VBITS_GE_128-LABEL: urem_v8i8: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ushll v2.8h, v1.8b, #0 ; VBITS_GE_128-NEXT: ushll v3.8h, v0.8b, #0 +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ushll2 v4.4s, v2.8h, #0 ; VBITS_GE_128-NEXT: ushll2 v5.4s, v3.8h, #0 ; VBITS_GE_128-NEXT: ushll v2.4s, v2.4h, #0 @@ -915,9 +915,9 @@ define <8 x i8> @urem_v8i8(<8 x i8> %op1, <8 x i8> %op2) #0 { define <16 x i8> @urem_v16i8(<16 x i8> %op1, <16 x i8> %op2) #0 { ; VBITS_GE_128-LABEL: urem_v16i8: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ushll2 v2.8h, v1.16b, #0 ; VBITS_GE_128-NEXT: ushll2 v3.8h, v0.16b, #0 +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ushll2 v4.4s, v2.8h, #0 ; VBITS_GE_128-NEXT: ushll2 v5.4s, v3.8h, #0 ; VBITS_GE_128-NEXT: ushll v2.4s, v2.4h, #0 @@ -1095,8 +1095,8 @@ define void @urem_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h ; CHECK-NEXT: udivr z3.s, p1/m, z3.s, z5.s ; CHECK-NEXT: ptrue p1.h, vl64 -; CHECK-NEXT: uzp1 z5.h, z6.h, z6.h ; CHECK-NEXT: splice z4.h, p1, z4.h, z2.h +; CHECK-NEXT: uzp1 z5.h, z6.h, z6.h ; CHECK-NEXT: uzp1 z2.b, z4.b, z4.b ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h ; CHECK-NEXT: splice z5.h, p1, z5.h, z3.h @@ -1118,9 +1118,9 @@ define void @urem_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 { define <4 x i16> @urem_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 { ; VBITS_GE_128-LABEL: urem_v4i16: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ushll v2.4s, v1.4h, #0 ; VBITS_GE_128-NEXT: ushll v3.4s, v0.4h, #0 +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: udivr z2.s, p0/m, z2.s, z3.s ; VBITS_GE_128-NEXT: xtn v2.4h, v2.4s ; VBITS_GE_128-NEXT: mls v0.4h, v2.4h, v1.4h @@ -1128,9 +1128,9 @@ define <4 x i16> @urem_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 { ; ; VBITS_GE_256-LABEL: urem_v4i16: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.s, vl4 ; VBITS_GE_256-NEXT: ushll v2.4s, v1.4h, #0 ; VBITS_GE_256-NEXT: ushll v3.4s, v0.4h, #0 +; VBITS_GE_256-NEXT: ptrue p0.s, vl4 ; VBITS_GE_256-NEXT: udivr z2.s, p0/m, z2.s, z3.s ; VBITS_GE_256-NEXT: mov w8, v2.s[1] ; VBITS_GE_256-NEXT: mov v3.16b, v2.16b @@ -1144,9 +1144,9 @@ define <4 x i16> @urem_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 { ; ; VBITS_GE_512-LABEL: urem_v4i16: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.s, vl4 ; VBITS_GE_512-NEXT: ushll v2.4s, v1.4h, #0 ; VBITS_GE_512-NEXT: ushll v3.4s, v0.4h, #0 +; VBITS_GE_512-NEXT: ptrue p0.s, vl4 ; VBITS_GE_512-NEXT: udivr z2.s, p0/m, z2.s, z3.s ; VBITS_GE_512-NEXT: mov w8, v2.s[1] ; VBITS_GE_512-NEXT: mov v3.16b, v2.16b @@ -1164,9 +1164,9 @@ define <4 x i16> @urem_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 { define <8 x i16> @urem_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 { ; VBITS_GE_128-LABEL: urem_v8i16: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ushll2 v2.4s, v1.8h, #0 ; VBITS_GE_128-NEXT: ushll2 v3.4s, v0.8h, #0 +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ushll v4.4s, v0.4h, #0 ; VBITS_GE_128-NEXT: udivr z2.s, p0/m, z2.s, z3.s ; VBITS_GE_128-NEXT: ushll v3.4s, v1.4h, #0 @@ -1177,11 +1177,11 @@ define <8 x i16> @urem_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 { ; ; VBITS_GE_256-LABEL: urem_v8i16: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: // kill: def $q1 killed $q1 def $z1 ; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 def $z0 ; VBITS_GE_256-NEXT: uunpklo z2.s, z1.h ; VBITS_GE_256-NEXT: uunpklo z3.s, z0.h +; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: udivr z2.s, p0/m, z2.s, z3.s ; VBITS_GE_256-NEXT: uzp1 z2.h, z2.h, z2.h ; VBITS_GE_256-NEXT: mls v0.8h, v2.8h, v1.8h @@ -1190,11 +1190,11 @@ define <8 x i16> @urem_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 { ; ; VBITS_GE_512-LABEL: urem_v8i16: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.s, vl8 ; VBITS_GE_512-NEXT: // kill: def $q1 killed $q1 def $z1 ; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 def $z0 ; VBITS_GE_512-NEXT: uunpklo z2.s, z1.h ; VBITS_GE_512-NEXT: uunpklo z3.s, z0.h +; VBITS_GE_512-NEXT: ptrue p0.s, vl8 ; VBITS_GE_512-NEXT: udivr z2.s, p0/m, z2.s, z3.s ; VBITS_GE_512-NEXT: uzp1 z2.h, z2.h, z2.h ; VBITS_GE_512-NEXT: mls v0.8h, v2.8h, v1.8h @@ -1207,8 +1207,8 @@ define <8 x i16> @urem_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 { define void @urem_v16i16(ptr %a, ptr %b) #0 { ; VBITS_GE_128-LABEL: urem_v16i16: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ldp q4, q1, [x1] +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ldr q0, [x0, #16] ; VBITS_GE_128-NEXT: ushll2 v2.4s, v1.8h, #0 ; VBITS_GE_128-NEXT: ushll2 v3.4s, v0.8h, #0 @@ -1400,25 +1400,25 @@ define void @urem_v8i32(ptr %a, ptr %b) vscale_range(2,0) #0 { define void @urem_v16i32(ptr %a, ptr %b) #0 { ; VBITS_GE_128-LABEL: urem_v16i32: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ldp q0, q3, [x1] +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ldp q1, q2, [x0] ; VBITS_GE_128-NEXT: ldp q16, q5, [x0, #32] ; VBITS_GE_128-NEXT: ldp q17, q6, [x1, #32] ; VBITS_GE_128-NEXT: movprfx z4, z1 ; VBITS_GE_128-NEXT: udiv z4.s, p0/m, z4.s, z0.s +; VBITS_GE_128-NEXT: movprfx z19, z2 +; VBITS_GE_128-NEXT: udiv z19.s, p0/m, z19.s, z3.s ; VBITS_GE_128-NEXT: movprfx z7, z5 ; VBITS_GE_128-NEXT: udiv z7.s, p0/m, z7.s, z6.s ; VBITS_GE_128-NEXT: movprfx z18, z16 ; VBITS_GE_128-NEXT: udiv z18.s, p0/m, z18.s, z17.s -; VBITS_GE_128-NEXT: movprfx z19, z2 -; VBITS_GE_128-NEXT: udiv z19.s, p0/m, z19.s, z3.s -; VBITS_GE_128-NEXT: mls v16.4s, v18.4s, v17.4s -; VBITS_GE_128-NEXT: mls v5.4s, v7.4s, v6.4s ; VBITS_GE_128-NEXT: mls v1.4s, v4.4s, v0.4s ; VBITS_GE_128-NEXT: mls v2.4s, v19.4s, v3.4s -; VBITS_GE_128-NEXT: stp q16, q5, [x0, #32] +; VBITS_GE_128-NEXT: mls v16.4s, v18.4s, v17.4s +; VBITS_GE_128-NEXT: mls v5.4s, v7.4s, v6.4s ; VBITS_GE_128-NEXT: stp q1, q2, [x0] +; VBITS_GE_128-NEXT: stp q16, q5, [x0, #32] ; VBITS_GE_128-NEXT: ret ; ; VBITS_GE_256-LABEL: urem_v16i32: @@ -1548,26 +1548,26 @@ define void @urem_v4i64(ptr %a, ptr %b) vscale_range(2,0) #0 { define void @urem_v8i64(ptr %a, ptr %b) #0 { ; VBITS_GE_128-LABEL: urem_v8i64: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.d, vl2 ; VBITS_GE_128-NEXT: ldp q0, q3, [x1] +; VBITS_GE_128-NEXT: ptrue p0.d, vl2 ; VBITS_GE_128-NEXT: ldp q1, q2, [x0] ; VBITS_GE_128-NEXT: ldp q16, q5, [x0, #32] ; VBITS_GE_128-NEXT: ldp q17, q6, [x1, #32] ; VBITS_GE_128-NEXT: movprfx z4, z1 ; VBITS_GE_128-NEXT: udiv z4.d, p0/m, z4.d, z0.d +; VBITS_GE_128-NEXT: movprfx z19, z2 +; VBITS_GE_128-NEXT: udiv z19.d, p0/m, z19.d, z3.d ; VBITS_GE_128-NEXT: movprfx z7, z5 ; VBITS_GE_128-NEXT: udiv z7.d, p0/m, z7.d, z6.d ; VBITS_GE_128-NEXT: movprfx z18, z16 ; VBITS_GE_128-NEXT: udiv z18.d, p0/m, z18.d, z17.d -; VBITS_GE_128-NEXT: movprfx z19, z2 -; VBITS_GE_128-NEXT: udiv z19.d, p0/m, z19.d, z3.d -; VBITS_GE_128-NEXT: mls z16.d, p0/m, z18.d, z17.d -; VBITS_GE_128-NEXT: mls z5.d, p0/m, z7.d, z6.d ; VBITS_GE_128-NEXT: msb z0.d, p0/m, z4.d, z1.d ; VBITS_GE_128-NEXT: movprfx z1, z2 ; VBITS_GE_128-NEXT: mls z1.d, p0/m, z19.d, z3.d -; VBITS_GE_128-NEXT: stp q16, q5, [x0, #32] +; VBITS_GE_128-NEXT: mls z16.d, p0/m, z18.d, z17.d +; VBITS_GE_128-NEXT: mls z5.d, p0/m, z7.d, z6.d ; VBITS_GE_128-NEXT: stp q0, q1, [x0] +; VBITS_GE_128-NEXT: stp q16, q5, [x0, #32] ; VBITS_GE_128-NEXT: ret ; ; VBITS_GE_256-LABEL: urem_v8i64: diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-select.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-select.ll index 710dce4de6dd..37396ba7011b 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-select.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-select.ll @@ -34,14 +34,14 @@ define <16 x i8> @select_v16i8(<16 x i8> %op1, <16 x i8> %op2, i1 %mask) vscale_ define void @select_v32i8(ptr %a, ptr %b, i1 %mask) vscale_range(2,0) #0 { ; CHECK-LABEL: select_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl32 ; CHECK-NEXT: mov z0.b, w2 -; CHECK-NEXT: ptrue p1.b -; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0] -; CHECK-NEXT: ld1b { z2.b }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.b, p1/z, z0.b, #0 -; CHECK-NEXT: sel z0.b, p1, z1.b, z2.b -; CHECK-NEXT: st1b { z0.b }, p0, [x0] +; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: ptrue p1.b, vl32 +; CHECK-NEXT: cmpne p0.b, p0/z, z0.b, #0 +; CHECK-NEXT: ld1b { z0.b }, p1/z, [x0] +; CHECK-NEXT: ld1b { z1.b }, p1/z, [x1] +; CHECK-NEXT: sel z0.b, p0, z0.b, z1.b +; CHECK-NEXT: st1b { z0.b }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <32 x i8>, ptr %a %op2 = load volatile <32 x i8>, ptr %b @@ -53,31 +53,31 @@ define void @select_v32i8(ptr %a, ptr %b, i1 %mask) vscale_range(2,0) #0 { define void @select_v64i8(ptr %a, ptr %b, i1 %mask) #0 { ; VBITS_GE_256-LABEL: select_v64i8: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.b, vl32 ; VBITS_GE_256-NEXT: mov z0.b, w2 +; VBITS_GE_256-NEXT: ptrue p0.b ; VBITS_GE_256-NEXT: mov w8, #32 // =0x20 -; VBITS_GE_256-NEXT: ptrue p1.b -; VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x0, x8] -; VBITS_GE_256-NEXT: ld1b { z2.b }, p0/z, [x0] -; VBITS_GE_256-NEXT: ld1b { z3.b }, p0/z, [x1, x8] -; VBITS_GE_256-NEXT: cmpne p1.b, p1/z, z0.b, #0 -; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x1] -; VBITS_GE_256-NEXT: sel z1.b, p1, z1.b, z3.b -; VBITS_GE_256-NEXT: mov z0.b, p1/m, z2.b -; VBITS_GE_256-NEXT: st1b { z1.b }, p0, [x0, x8] -; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0] +; VBITS_GE_256-NEXT: ptrue p1.b, vl32 +; VBITS_GE_256-NEXT: cmpne p0.b, p0/z, z0.b, #0 +; VBITS_GE_256-NEXT: ld1b { z0.b }, p1/z, [x0, x8] +; VBITS_GE_256-NEXT: ld1b { z1.b }, p1/z, [x0] +; VBITS_GE_256-NEXT: ld1b { z2.b }, p1/z, [x1, x8] +; VBITS_GE_256-NEXT: ld1b { z3.b }, p1/z, [x1] +; VBITS_GE_256-NEXT: sel z0.b, p0, z0.b, z2.b +; VBITS_GE_256-NEXT: sel z1.b, p0, z1.b, z3.b +; VBITS_GE_256-NEXT: st1b { z0.b }, p1, [x0, x8] +; VBITS_GE_256-NEXT: st1b { z1.b }, p1, [x0] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: select_v64i8: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.b, vl64 ; VBITS_GE_512-NEXT: mov z0.b, w2 -; VBITS_GE_512-NEXT: ptrue p1.b -; VBITS_GE_512-NEXT: ld1b { z1.b }, p0/z, [x0] -; VBITS_GE_512-NEXT: ld1b { z2.b }, p0/z, [x1] -; VBITS_GE_512-NEXT: cmpne p1.b, p1/z, z0.b, #0 -; VBITS_GE_512-NEXT: sel z0.b, p1, z1.b, z2.b -; VBITS_GE_512-NEXT: st1b { z0.b }, p0, [x0] +; VBITS_GE_512-NEXT: ptrue p0.b +; VBITS_GE_512-NEXT: ptrue p1.b, vl64 +; VBITS_GE_512-NEXT: cmpne p0.b, p0/z, z0.b, #0 +; VBITS_GE_512-NEXT: ld1b { z0.b }, p1/z, [x0] +; VBITS_GE_512-NEXT: ld1b { z1.b }, p1/z, [x1] +; VBITS_GE_512-NEXT: sel z0.b, p0, z0.b, z1.b +; VBITS_GE_512-NEXT: st1b { z0.b }, p1, [x0] ; VBITS_GE_512-NEXT: ret %op1 = load volatile <64 x i8>, ptr %a %op2 = load volatile <64 x i8>, ptr %b @@ -89,14 +89,14 @@ define void @select_v64i8(ptr %a, ptr %b, i1 %mask) #0 { define void @select_v128i8(ptr %a, ptr %b, i1 %mask) vscale_range(8,0) #0 { ; CHECK-LABEL: select_v128i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl128 ; CHECK-NEXT: mov z0.b, w2 -; CHECK-NEXT: ptrue p1.b -; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0] -; CHECK-NEXT: ld1b { z2.b }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.b, p1/z, z0.b, #0 -; CHECK-NEXT: sel z0.b, p1, z1.b, z2.b -; CHECK-NEXT: st1b { z0.b }, p0, [x0] +; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: ptrue p1.b, vl128 +; CHECK-NEXT: cmpne p0.b, p0/z, z0.b, #0 +; CHECK-NEXT: ld1b { z0.b }, p1/z, [x0] +; CHECK-NEXT: ld1b { z1.b }, p1/z, [x1] +; CHECK-NEXT: sel z0.b, p0, z0.b, z1.b +; CHECK-NEXT: st1b { z0.b }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <128 x i8>, ptr %a %op2 = load volatile <128 x i8>, ptr %b @@ -108,14 +108,14 @@ define void @select_v128i8(ptr %a, ptr %b, i1 %mask) vscale_range(8,0) #0 { define void @select_v256i8(ptr %a, ptr %b, i1 %mask) vscale_range(16,0) #0 { ; CHECK-LABEL: select_v256i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl256 ; CHECK-NEXT: mov z0.b, w2 -; CHECK-NEXT: ptrue p1.b -; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0] -; CHECK-NEXT: ld1b { z2.b }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.b, p1/z, z0.b, #0 -; CHECK-NEXT: sel z0.b, p1, z1.b, z2.b -; CHECK-NEXT: st1b { z0.b }, p0, [x0] +; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: ptrue p1.b, vl256 +; CHECK-NEXT: cmpne p0.b, p0/z, z0.b, #0 +; CHECK-NEXT: ld1b { z0.b }, p1/z, [x0] +; CHECK-NEXT: ld1b { z1.b }, p1/z, [x1] +; CHECK-NEXT: sel z0.b, p0, z0.b, z1.b +; CHECK-NEXT: st1b { z0.b }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <256 x i8>, ptr %a %op2 = load volatile <256 x i8>, ptr %b @@ -153,15 +153,15 @@ define <8 x i16> @select_v8i16(<8 x i16> %op1, <8 x i16> %op2, i1 %mask) vscale_ define void @select_v16i16(ptr %a, ptr %b, i1 %mask) vscale_range(2,0) #0 { ; CHECK-LABEL: select_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl16 ; CHECK-NEXT: mov z0.h, w2 -; CHECK-NEXT: ptrue p1.h +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: ptrue p1.h, vl16 ; CHECK-NEXT: and z0.h, z0.h, #0x1 -; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0] -; CHECK-NEXT: ld1h { z2.h }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.h, p1/z, z0.h, #0 -; CHECK-NEXT: sel z0.h, p1, z1.h, z2.h -; CHECK-NEXT: st1h { z0.h }, p0, [x0] +; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #0 +; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0] +; CHECK-NEXT: ld1h { z1.h }, p1/z, [x1] +; CHECK-NEXT: sel z0.h, p0, z0.h, z1.h +; CHECK-NEXT: st1h { z0.h }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <16 x i16>, ptr %a %op2 = load volatile <16 x i16>, ptr %b @@ -173,33 +173,33 @@ define void @select_v16i16(ptr %a, ptr %b, i1 %mask) vscale_range(2,0) #0 { define void @select_v32i16(ptr %a, ptr %b, i1 %mask) #0 { ; VBITS_GE_256-LABEL: select_v32i16: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.h, vl16 ; VBITS_GE_256-NEXT: mov z0.h, w2 +; VBITS_GE_256-NEXT: ptrue p0.h ; VBITS_GE_256-NEXT: mov x8, #16 // =0x10 -; VBITS_GE_256-NEXT: ptrue p1.h +; VBITS_GE_256-NEXT: ptrue p1.h, vl16 ; VBITS_GE_256-NEXT: and z0.h, z0.h, #0x1 -; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x0, x8, lsl #1] -; VBITS_GE_256-NEXT: ld1h { z2.h }, p0/z, [x0] -; VBITS_GE_256-NEXT: ld1h { z3.h }, p0/z, [x1, x8, lsl #1] -; VBITS_GE_256-NEXT: cmpne p1.h, p1/z, z0.h, #0 -; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x1] -; VBITS_GE_256-NEXT: sel z1.h, p1, z1.h, z3.h -; VBITS_GE_256-NEXT: mov z0.h, p1/m, z2.h -; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x0, x8, lsl #1] -; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0] +; VBITS_GE_256-NEXT: cmpne p0.h, p0/z, z0.h, #0 +; VBITS_GE_256-NEXT: ld1h { z0.h }, p1/z, [x0, x8, lsl #1] +; VBITS_GE_256-NEXT: ld1h { z1.h }, p1/z, [x0] +; VBITS_GE_256-NEXT: ld1h { z2.h }, p1/z, [x1, x8, lsl #1] +; VBITS_GE_256-NEXT: ld1h { z3.h }, p1/z, [x1] +; VBITS_GE_256-NEXT: sel z0.h, p0, z0.h, z2.h +; VBITS_GE_256-NEXT: sel z1.h, p0, z1.h, z3.h +; VBITS_GE_256-NEXT: st1h { z0.h }, p1, [x0, x8, lsl #1] +; VBITS_GE_256-NEXT: st1h { z1.h }, p1, [x0] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: select_v32i16: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.h, vl32 ; VBITS_GE_512-NEXT: mov z0.h, w2 -; VBITS_GE_512-NEXT: ptrue p1.h +; VBITS_GE_512-NEXT: ptrue p0.h +; VBITS_GE_512-NEXT: ptrue p1.h, vl32 ; VBITS_GE_512-NEXT: and z0.h, z0.h, #0x1 -; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x0] -; VBITS_GE_512-NEXT: ld1h { z2.h }, p0/z, [x1] -; VBITS_GE_512-NEXT: cmpne p1.h, p1/z, z0.h, #0 -; VBITS_GE_512-NEXT: sel z0.h, p1, z1.h, z2.h -; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0] +; VBITS_GE_512-NEXT: cmpne p0.h, p0/z, z0.h, #0 +; VBITS_GE_512-NEXT: ld1h { z0.h }, p1/z, [x0] +; VBITS_GE_512-NEXT: ld1h { z1.h }, p1/z, [x1] +; VBITS_GE_512-NEXT: sel z0.h, p0, z0.h, z1.h +; VBITS_GE_512-NEXT: st1h { z0.h }, p1, [x0] ; VBITS_GE_512-NEXT: ret %op1 = load volatile <32 x i16>, ptr %a %op2 = load volatile <32 x i16>, ptr %b @@ -211,15 +211,15 @@ define void @select_v32i16(ptr %a, ptr %b, i1 %mask) #0 { define void @select_v64i16(ptr %a, ptr %b, i1 %mask) vscale_range(8,0) #0 { ; CHECK-LABEL: select_v64i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl64 ; CHECK-NEXT: mov z0.h, w2 -; CHECK-NEXT: ptrue p1.h +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: ptrue p1.h, vl64 ; CHECK-NEXT: and z0.h, z0.h, #0x1 -; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0] -; CHECK-NEXT: ld1h { z2.h }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.h, p1/z, z0.h, #0 -; CHECK-NEXT: sel z0.h, p1, z1.h, z2.h -; CHECK-NEXT: st1h { z0.h }, p0, [x0] +; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #0 +; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0] +; CHECK-NEXT: ld1h { z1.h }, p1/z, [x1] +; CHECK-NEXT: sel z0.h, p0, z0.h, z1.h +; CHECK-NEXT: st1h { z0.h }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <64 x i16>, ptr %a %op2 = load volatile <64 x i16>, ptr %b @@ -231,15 +231,15 @@ define void @select_v64i16(ptr %a, ptr %b, i1 %mask) vscale_range(8,0) #0 { define void @select_v128i16(ptr %a, ptr %b, i1 %mask) vscale_range(16,0) #0 { ; CHECK-LABEL: select_v128i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl128 ; CHECK-NEXT: mov z0.h, w2 -; CHECK-NEXT: ptrue p1.h +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: ptrue p1.h, vl128 ; CHECK-NEXT: and z0.h, z0.h, #0x1 -; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0] -; CHECK-NEXT: ld1h { z2.h }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.h, p1/z, z0.h, #0 -; CHECK-NEXT: sel z0.h, p1, z1.h, z2.h -; CHECK-NEXT: st1h { z0.h }, p0, [x0] +; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #0 +; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0] +; CHECK-NEXT: ld1h { z1.h }, p1/z, [x1] +; CHECK-NEXT: sel z0.h, p0, z0.h, z1.h +; CHECK-NEXT: st1h { z0.h }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <128 x i16>, ptr %a %op2 = load volatile <128 x i16>, ptr %b @@ -277,15 +277,15 @@ define <4 x i32> @select_v4i32(<4 x i32> %op1, <4 x i32> %op2, i1 %mask) vscale_ define void @select_v8i32(ptr %a, ptr %b, i1 %mask) vscale_range(2,0) #0 { ; CHECK-LABEL: select_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: and w8, w2, #0x1 -; CHECK-NEXT: ptrue p1.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z0.s, w8 -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.s, p1/z, z0.s, #0 -; CHECK-NEXT: sel z0.s, p1, z1.s, z2.s -; CHECK-NEXT: st1w { z0.s }, p0, [x0] +; CHECK-NEXT: ptrue p1.s, vl8 +; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0 +; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0] +; CHECK-NEXT: ld1w { z1.s }, p1/z, [x1] +; CHECK-NEXT: sel z0.s, p0, z0.s, z1.s +; CHECK-NEXT: st1w { z0.s }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <8 x i32>, ptr %a %op2 = load volatile <8 x i32>, ptr %b @@ -297,33 +297,33 @@ define void @select_v8i32(ptr %a, ptr %b, i1 %mask) vscale_range(2,0) #0 { define void @select_v16i32(ptr %a, ptr %b, i1 %mask) #0 { ; VBITS_GE_256-LABEL: select_v16i32: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: and w8, w2, #0x1 -; VBITS_GE_256-NEXT: ptrue p1.s +; VBITS_GE_256-NEXT: ptrue p0.s ; VBITS_GE_256-NEXT: mov z0.s, w8 +; VBITS_GE_256-NEXT: ptrue p1.s, vl8 ; VBITS_GE_256-NEXT: mov x8, #8 // =0x8 -; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0, x8, lsl #2] -; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x0] -; VBITS_GE_256-NEXT: ld1w { z3.s }, p0/z, [x1, x8, lsl #2] -; VBITS_GE_256-NEXT: cmpne p1.s, p1/z, z0.s, #0 -; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x1] -; VBITS_GE_256-NEXT: sel z1.s, p1, z1.s, z3.s -; VBITS_GE_256-NEXT: mov z0.s, p1/m, z2.s -; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0, x8, lsl #2] -; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0] +; VBITS_GE_256-NEXT: cmpne p0.s, p0/z, z0.s, #0 +; VBITS_GE_256-NEXT: ld1w { z0.s }, p1/z, [x0, x8, lsl #2] +; VBITS_GE_256-NEXT: ld1w { z1.s }, p1/z, [x0] +; VBITS_GE_256-NEXT: ld1w { z2.s }, p1/z, [x1, x8, lsl #2] +; VBITS_GE_256-NEXT: ld1w { z3.s }, p1/z, [x1] +; VBITS_GE_256-NEXT: sel z0.s, p0, z0.s, z2.s +; VBITS_GE_256-NEXT: sel z1.s, p0, z1.s, z3.s +; VBITS_GE_256-NEXT: st1w { z0.s }, p1, [x0, x8, lsl #2] +; VBITS_GE_256-NEXT: st1w { z1.s }, p1, [x0] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: select_v16i32: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.s, vl16 ; VBITS_GE_512-NEXT: and w8, w2, #0x1 -; VBITS_GE_512-NEXT: ptrue p1.s +; VBITS_GE_512-NEXT: ptrue p0.s ; VBITS_GE_512-NEXT: mov z0.s, w8 -; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x0] -; VBITS_GE_512-NEXT: ld1w { z2.s }, p0/z, [x1] -; VBITS_GE_512-NEXT: cmpne p1.s, p1/z, z0.s, #0 -; VBITS_GE_512-NEXT: sel z0.s, p1, z1.s, z2.s -; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0] +; VBITS_GE_512-NEXT: ptrue p1.s, vl16 +; VBITS_GE_512-NEXT: cmpne p0.s, p0/z, z0.s, #0 +; VBITS_GE_512-NEXT: ld1w { z0.s }, p1/z, [x0] +; VBITS_GE_512-NEXT: ld1w { z1.s }, p1/z, [x1] +; VBITS_GE_512-NEXT: sel z0.s, p0, z0.s, z1.s +; VBITS_GE_512-NEXT: st1w { z0.s }, p1, [x0] ; VBITS_GE_512-NEXT: ret %op1 = load volatile <16 x i32>, ptr %a %op2 = load volatile <16 x i32>, ptr %b @@ -335,15 +335,15 @@ define void @select_v16i32(ptr %a, ptr %b, i1 %mask) #0 { define void @select_v32i32(ptr %a, ptr %b, i1 %mask) vscale_range(8,0) #0 { ; CHECK-LABEL: select_v32i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl32 ; CHECK-NEXT: and w8, w2, #0x1 -; CHECK-NEXT: ptrue p1.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z0.s, w8 -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.s, p1/z, z0.s, #0 -; CHECK-NEXT: sel z0.s, p1, z1.s, z2.s -; CHECK-NEXT: st1w { z0.s }, p0, [x0] +; CHECK-NEXT: ptrue p1.s, vl32 +; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0 +; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0] +; CHECK-NEXT: ld1w { z1.s }, p1/z, [x1] +; CHECK-NEXT: sel z0.s, p0, z0.s, z1.s +; CHECK-NEXT: st1w { z0.s }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <32 x i32>, ptr %a %op2 = load volatile <32 x i32>, ptr %b @@ -355,15 +355,15 @@ define void @select_v32i32(ptr %a, ptr %b, i1 %mask) vscale_range(8,0) #0 { define void @select_v64i32(ptr %a, ptr %b, i1 %mask) vscale_range(16,0) #0 { ; CHECK-LABEL: select_v64i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl64 ; CHECK-NEXT: and w8, w2, #0x1 -; CHECK-NEXT: ptrue p1.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z0.s, w8 -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.s, p1/z, z0.s, #0 -; CHECK-NEXT: sel z0.s, p1, z1.s, z2.s -; CHECK-NEXT: st1w { z0.s }, p0, [x0] +; CHECK-NEXT: ptrue p1.s, vl64 +; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0 +; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0] +; CHECK-NEXT: ld1w { z1.s }, p1/z, [x1] +; CHECK-NEXT: sel z0.s, p0, z0.s, z1.s +; CHECK-NEXT: st1w { z0.s }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <64 x i32>, ptr %a %op2 = load volatile <64 x i32>, ptr %b @@ -401,16 +401,16 @@ define <2 x i64> @select_v2i64(<2 x i64> %op1, <2 x i64> %op2, i1 %mask) vscale_ define void @select_v4i64(ptr %a, ptr %b, i1 %mask) vscale_range(2,0) #0 { ; CHECK-LABEL: select_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: // kill: def $w2 killed $w2 def $x2 ; CHECK-NEXT: and x8, x2, #0x1 -; CHECK-NEXT: ptrue p1.d +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z0.d, x8 -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.d, p1/z, z0.d, #0 -; CHECK-NEXT: sel z0.d, p1, z1.d, z2.d -; CHECK-NEXT: st1d { z0.d }, p0, [x0] +; CHECK-NEXT: ptrue p1.d, vl4 +; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 +; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0] +; CHECK-NEXT: ld1d { z1.d }, p1/z, [x1] +; CHECK-NEXT: sel z0.d, p0, z0.d, z1.d +; CHECK-NEXT: st1d { z0.d }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <4 x i64>, ptr %a %op2 = load volatile <4 x i64>, ptr %b @@ -422,35 +422,35 @@ define void @select_v4i64(ptr %a, ptr %b, i1 %mask) vscale_range(2,0) #0 { define void @select_v8i64(ptr %a, ptr %b, i1 %mask) #0 { ; VBITS_GE_256-LABEL: select_v8i64: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: // kill: def $w2 killed $w2 def $x2 ; VBITS_GE_256-NEXT: and x8, x2, #0x1 -; VBITS_GE_256-NEXT: ptrue p1.d +; VBITS_GE_256-NEXT: ptrue p0.d ; VBITS_GE_256-NEXT: mov z0.d, x8 +; VBITS_GE_256-NEXT: ptrue p1.d, vl4 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 -; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0, x8, lsl #3] -; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x0] -; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x1, x8, lsl #3] -; VBITS_GE_256-NEXT: cmpne p1.d, p1/z, z0.d, #0 -; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1] -; VBITS_GE_256-NEXT: sel z1.d, p1, z1.d, z3.d -; VBITS_GE_256-NEXT: mov z0.d, p1/m, z2.d -; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0, x8, lsl #3] -; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0] +; VBITS_GE_256-NEXT: cmpne p0.d, p0/z, z0.d, #0 +; VBITS_GE_256-NEXT: ld1d { z0.d }, p1/z, [x0, x8, lsl #3] +; VBITS_GE_256-NEXT: ld1d { z1.d }, p1/z, [x0] +; VBITS_GE_256-NEXT: ld1d { z2.d }, p1/z, [x1, x8, lsl #3] +; VBITS_GE_256-NEXT: ld1d { z3.d }, p1/z, [x1] +; VBITS_GE_256-NEXT: sel z0.d, p0, z0.d, z2.d +; VBITS_GE_256-NEXT: sel z1.d, p0, z1.d, z3.d +; VBITS_GE_256-NEXT: st1d { z0.d }, p1, [x0, x8, lsl #3] +; VBITS_GE_256-NEXT: st1d { z1.d }, p1, [x0] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: select_v8i64: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.d, vl8 ; VBITS_GE_512-NEXT: // kill: def $w2 killed $w2 def $x2 ; VBITS_GE_512-NEXT: and x8, x2, #0x1 -; VBITS_GE_512-NEXT: ptrue p1.d +; VBITS_GE_512-NEXT: ptrue p0.d ; VBITS_GE_512-NEXT: mov z0.d, x8 -; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x0] -; VBITS_GE_512-NEXT: ld1d { z2.d }, p0/z, [x1] -; VBITS_GE_512-NEXT: cmpne p1.d, p1/z, z0.d, #0 -; VBITS_GE_512-NEXT: sel z0.d, p1, z1.d, z2.d -; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0] +; VBITS_GE_512-NEXT: ptrue p1.d, vl8 +; VBITS_GE_512-NEXT: cmpne p0.d, p0/z, z0.d, #0 +; VBITS_GE_512-NEXT: ld1d { z0.d }, p1/z, [x0] +; VBITS_GE_512-NEXT: ld1d { z1.d }, p1/z, [x1] +; VBITS_GE_512-NEXT: sel z0.d, p0, z0.d, z1.d +; VBITS_GE_512-NEXT: st1d { z0.d }, p1, [x0] ; VBITS_GE_512-NEXT: ret %op1 = load volatile <8 x i64>, ptr %a %op2 = load volatile <8 x i64>, ptr %b @@ -462,16 +462,16 @@ define void @select_v8i64(ptr %a, ptr %b, i1 %mask) #0 { define void @select_v16i64(ptr %a, ptr %b, i1 %mask) vscale_range(8,0) #0 { ; CHECK-LABEL: select_v16i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl16 ; CHECK-NEXT: // kill: def $w2 killed $w2 def $x2 ; CHECK-NEXT: and x8, x2, #0x1 -; CHECK-NEXT: ptrue p1.d +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z0.d, x8 -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.d, p1/z, z0.d, #0 -; CHECK-NEXT: sel z0.d, p1, z1.d, z2.d -; CHECK-NEXT: st1d { z0.d }, p0, [x0] +; CHECK-NEXT: ptrue p1.d, vl16 +; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 +; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0] +; CHECK-NEXT: ld1d { z1.d }, p1/z, [x1] +; CHECK-NEXT: sel z0.d, p0, z0.d, z1.d +; CHECK-NEXT: st1d { z0.d }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <16 x i64>, ptr %a %op2 = load volatile <16 x i64>, ptr %b @@ -483,16 +483,16 @@ define void @select_v16i64(ptr %a, ptr %b, i1 %mask) vscale_range(8,0) #0 { define void @select_v32i64(ptr %a, ptr %b, i1 %mask) vscale_range(16,0) #0 { ; CHECK-LABEL: select_v32i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl32 ; CHECK-NEXT: // kill: def $w2 killed $w2 def $x2 ; CHECK-NEXT: and x8, x2, #0x1 -; CHECK-NEXT: ptrue p1.d +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z0.d, x8 -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.d, p1/z, z0.d, #0 -; CHECK-NEXT: sel z0.d, p1, z1.d, z2.d -; CHECK-NEXT: st1d { z0.d }, p0, [x0] +; CHECK-NEXT: ptrue p1.d, vl32 +; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 +; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0] +; CHECK-NEXT: ld1d { z1.d }, p1/z, [x1] +; CHECK-NEXT: sel z0.d, p0, z0.d, z1.d +; CHECK-NEXT: st1d { z0.d }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <32 x i64>, ptr %a %op2 = load volatile <32 x i64>, ptr %b diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-to-fp.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-to-fp.ll index 50040eaa61e6..5bb012ae5750 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-to-fp.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-to-fp.ll @@ -131,8 +131,8 @@ define <4 x float> @ucvtf_v4i16_v4f32(<4 x i16> %op1) vscale_range(2,0) #0 { define void @ucvtf_v8i16_v8f32(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: ucvtf_v8i16_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: ldr q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: ucvtf z0.s, p0/m, z0.s ; CHECK-NEXT: st1w { z0.s }, p0, [x1] @@ -354,7 +354,6 @@ define void @ucvtf_v16i32_v16f16(ptr %a, ptr %b) #0 { ; VBITS_GE_256: // %bb.0: ; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: mov x8, #8 // =0x8 -; VBITS_GE_256-NEXT: ptrue p1.h, vl16 ; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2] ; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0] ; VBITS_GE_256-NEXT: ucvtf z0.h, p0/m, z0.s @@ -363,7 +362,8 @@ define void @ucvtf_v16i32_v16f16(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h ; VBITS_GE_256-NEXT: splice z1.h, p0, z1.h, z0.h -; VBITS_GE_256-NEXT: st1h { z1.h }, p1, [x1] +; VBITS_GE_256-NEXT: ptrue p0.h, vl16 +; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x1] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: ucvtf_v16i32_v16f16: @@ -535,8 +535,8 @@ define <2 x double> @ucvtf_v2i32_v2f64(<2 x i32> %op1) vscale_range(2,0) #0 { define void @ucvtf_v4i32_v4f64(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: ucvtf_v4i32_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: ldr q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: ucvtf z0.d, p0/m, z0.d ; CHECK-NEXT: st1d { z0.d }, p0, [x1] @@ -759,7 +759,6 @@ define void @ucvtf_v8i64_v8f32(ptr %a, ptr %b) #0 { ; VBITS_GE_256: // %bb.0: ; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 -; VBITS_GE_256-NEXT: ptrue p1.s, vl8 ; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] ; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0] ; VBITS_GE_256-NEXT: ucvtf z0.s, p0/m, z0.d @@ -768,7 +767,8 @@ define void @ucvtf_v8i64_v8f32(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s ; VBITS_GE_256-NEXT: splice z1.s, p0, z1.s, z0.s -; VBITS_GE_256-NEXT: st1w { z1.s }, p1, [x1] +; VBITS_GE_256-NEXT: ptrue p0.s, vl8 +; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x1] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: ucvtf_v8i64_v8f32: @@ -1038,8 +1038,8 @@ define <4 x float> @scvtf_v4i16_v4f32(<4 x i16> %op1) vscale_range(2,0) #0 { define void @scvtf_v8i16_v8f32(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: scvtf_v8i16_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: ldr q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: sunpklo z0.s, z0.h ; CHECK-NEXT: scvtf z0.s, p0/m, z0.s ; CHECK-NEXT: st1w { z0.s }, p0, [x1] @@ -1273,7 +1273,6 @@ define void @scvtf_v16i32_v16f16(ptr %a, ptr %b) #0 { ; VBITS_GE_256: // %bb.0: ; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: mov x8, #8 // =0x8 -; VBITS_GE_256-NEXT: ptrue p1.h, vl16 ; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2] ; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0] ; VBITS_GE_256-NEXT: scvtf z0.h, p0/m, z0.s @@ -1282,7 +1281,8 @@ define void @scvtf_v16i32_v16f16(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h ; VBITS_GE_256-NEXT: splice z1.h, p0, z1.h, z0.h -; VBITS_GE_256-NEXT: st1h { z1.h }, p1, [x1] +; VBITS_GE_256-NEXT: ptrue p0.h, vl16 +; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x1] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: scvtf_v16i32_v16f16: @@ -1454,8 +1454,8 @@ define <2 x double> @scvtf_v2i32_v2f64(<2 x i32> %op1) vscale_range(2,0) #0 { define void @scvtf_v4i32_v4f64(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: scvtf_v4i32_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: ldr q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: sunpklo z0.d, z0.s ; CHECK-NEXT: scvtf z0.d, p0/m, z0.d ; CHECK-NEXT: st1d { z0.d }, p0, [x1] @@ -1684,7 +1684,6 @@ define void @scvtf_v8i64_v8f32(ptr %a, ptr %b) #0 { ; VBITS_GE_256: // %bb.0: ; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 -; VBITS_GE_256-NEXT: ptrue p1.s, vl8 ; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] ; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0] ; VBITS_GE_256-NEXT: scvtf z0.s, p0/m, z0.d @@ -1693,7 +1692,8 @@ define void @scvtf_v8i64_v8f32(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s ; VBITS_GE_256-NEXT: splice z1.s, p0, z1.s, z0.s -; VBITS_GE_256-NEXT: st1w { z1.s }, p1, [x1] +; VBITS_GE_256-NEXT: ptrue p0.s, vl8 +; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x1] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: scvtf_v8i64_v8f32: diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-mask-opt.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-mask-opt.ll index e23151475014..f2ad98f8caec 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-mask-opt.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-mask-opt.ll @@ -246,16 +246,16 @@ define void @masked_gather_v8i32(ptr %a, ptr %b) #0 { ; VBITS_GE_256: // %bb.0: ; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 -; VBITS_GE_256-NEXT: ptrue p1.s, vl8 ; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1, x8, lsl #3] ; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1] ; VBITS_GE_256-NEXT: ld1w { z0.d }, p0/z, [z0.d] ; VBITS_GE_256-NEXT: ld1w { z1.d }, p0/z, [z1.d] -; VBITS_GE_256-NEXT: ptrue p0.s, vl4 ; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s +; VBITS_GE_256-NEXT: ptrue p0.s, vl4 ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s ; VBITS_GE_256-NEXT: splice z1.s, p0, z1.s, z0.s -; VBITS_GE_256-NEXT: st1w { z1.s }, p1, [x0] +; VBITS_GE_256-NEXT: ptrue p0.s, vl8 +; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: masked_gather_v8i32: diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-128bit-loads.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-128bit-loads.ll index 67a53d4e15f3..55d37d1bda5e 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-128bit-loads.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-128bit-loads.ll @@ -11,8 +11,8 @@ target triple = "aarch64-unknown-linux-gnu" define <16 x i8> @masked_load_v16i8(ptr %src, <16 x i1> %mask) { ; CHECK-LABEL: masked_load_v16i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: shl v0.16b, v0.16b, #7 +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: cmlt v0.16b, v0.16b, #0 ; CHECK-NEXT: cmpne p0.b, p0/z, z0.b, #0 ; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0] diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-128bit-stores.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-128bit-stores.ll index bdd6ce064701..1a19b77f53c6 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-128bit-stores.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-128bit-stores.ll @@ -11,12 +11,12 @@ target triple = "aarch64-unknown-linux-gnu" define void @masked_store_v16i8(ptr %dst, <16 x i1> %mask) { ; CHECK-LABEL: masked_store_v16i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: shl v0.16b, v0.16b, #7 -; CHECK-NEXT: movi v1.2d, #0000000000000000 +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: cmlt v0.16b, v0.16b, #0 ; CHECK-NEXT: cmpne p0.b, p0/z, z0.b, #0 -; CHECK-NEXT: st1b { z1.b }, p0, [x0] +; CHECK-NEXT: movi v0.2d, #0000000000000000 +; CHECK-NEXT: st1b { z0.b }, p0, [x0] ; CHECK-NEXT: ret call void @llvm.masked.store.v16i8(<16 x i8> zeroinitializer, ptr %dst, i32 8, <16 x i1> %mask) ret void @@ -27,11 +27,11 @@ define void @masked_store_v8f16(ptr %dst, <8 x i1> %mask) { ; CHECK: // %bb.0: ; CHECK-NEXT: ushll v0.8h, v0.8b, #0 ; CHECK-NEXT: ptrue p0.h, vl8 -; CHECK-NEXT: movi v1.2d, #0000000000000000 ; CHECK-NEXT: shl v0.8h, v0.8h, #15 ; CHECK-NEXT: cmlt v0.8h, v0.8h, #0 ; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #0 -; CHECK-NEXT: st1h { z1.h }, p0, [x0] +; CHECK-NEXT: movi v0.2d, #0000000000000000 +; CHECK-NEXT: st1h { z0.h }, p0, [x0] ; CHECK-NEXT: ret call void @llvm.masked.store.v8f16(<8 x half> zeroinitializer, ptr %dst, i32 8, <8 x i1> %mask) ret void @@ -42,11 +42,11 @@ define void @masked_store_v4f32(ptr %dst, <4 x i1> %mask) { ; CHECK: // %bb.0: ; CHECK-NEXT: ushll v0.4s, v0.4h, #0 ; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: movi v1.2d, #0000000000000000 ; CHECK-NEXT: shl v0.4s, v0.4s, #31 ; CHECK-NEXT: cmlt v0.4s, v0.4s, #0 ; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0 -; CHECK-NEXT: st1w { z1.s }, p0, [x0] +; CHECK-NEXT: movi v0.2d, #0000000000000000 +; CHECK-NEXT: st1w { z0.s }, p0, [x0] ; CHECK-NEXT: ret call void @llvm.masked.store.v4f32(<4 x float> zeroinitializer, ptr %dst, i32 8, <4 x i1> %mask) ret void @@ -57,11 +57,11 @@ define void @masked_store_v2f64(ptr %dst, <2 x i1> %mask) { ; CHECK: // %bb.0: ; CHECK-NEXT: ushll v0.2d, v0.2s, #0 ; CHECK-NEXT: ptrue p0.d, vl2 -; CHECK-NEXT: movi v1.2d, #0000000000000000 ; CHECK-NEXT: shl v0.2d, v0.2d, #63 ; CHECK-NEXT: cmlt v0.2d, v0.2d, #0 ; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 -; CHECK-NEXT: st1d { z1.d }, p0, [x0] +; CHECK-NEXT: movi v0.2d, #0000000000000000 +; CHECK-NEXT: st1d { z0.d }, p0, [x0] ; CHECK-NEXT: ret call void @llvm.masked.store.v2f64(<2 x double> zeroinitializer, ptr %dst, i32 8, <2 x i1> %mask) ret void diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-gather.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-gather.ll index 92fce4584f6a..27e95489f8ad 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-gather.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-gather.ll @@ -41,11 +41,11 @@ define void @masked_gather_v4i8(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: ushll v0.8h, v0.8b, #0 ; CHECK-NEXT: cmeq v0.4h, v0.4h, #0 -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] ; CHECK-NEXT: sunpklo z0.s, z0.h ; CHECK-NEXT: sunpklo z0.d, z0.s ; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0 -; CHECK-NEXT: ld1b { z0.d }, p1/z, [z1.d] +; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1] +; CHECK-NEXT: ld1b { z0.d }, p1/z, [z0.d] ; CHECK-NEXT: st1b { z0.d }, p0, [x0] ; CHECK-NEXT: ret %cval = load <4 x i8>, ptr %a @@ -65,7 +65,6 @@ define void @masked_gather_v8i8(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: cmeq v0.8b, v0.8b, #0 ; VBITS_GE_256-NEXT: zip2 v1.8b, v0.8b, v0.8b ; VBITS_GE_256-NEXT: zip1 v0.8b, v0.8b, v0.8b -; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3] ; VBITS_GE_256-NEXT: shl v1.4h, v1.4h, #8 ; VBITS_GE_256-NEXT: shl v0.4h, v0.4h, #8 ; VBITS_GE_256-NEXT: sshr v1.4h, v1.4h, #8 @@ -75,10 +74,11 @@ define void @masked_gather_v8i8(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s ; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s ; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z1.d, #0 -; VBITS_GE_256-NEXT: ld1b { z1.d }, p1/z, [z2.d] -; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1] -; VBITS_GE_256-NEXT: cmpne p0.d, p0/z, z0.d, #0 -; VBITS_GE_256-NEXT: ld1b { z0.d }, p0/z, [z2.d] +; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1, x8, lsl #3] +; VBITS_GE_256-NEXT: ld1b { z1.d }, p1/z, [z1.d] +; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z0.d, #0 +; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1] +; VBITS_GE_256-NEXT: ld1b { z0.d }, p1/z, [z0.d] ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s ; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h ; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s @@ -93,11 +93,11 @@ define void @masked_gather_v8i8(ptr %a, ptr %b) #0 { ; VBITS_GE_512-NEXT: ptrue p0.d, vl8 ; VBITS_GE_512-NEXT: cmeq v0.8b, v0.8b, #0 ; VBITS_GE_512-NEXT: sunpklo z0.h, z0.b -; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1] ; VBITS_GE_512-NEXT: sunpklo z0.s, z0.h ; VBITS_GE_512-NEXT: sunpklo z0.d, z0.s -; VBITS_GE_512-NEXT: cmpne p0.d, p0/z, z0.d, #0 -; VBITS_GE_512-NEXT: ld1b { z0.d }, p0/z, [z1.d] +; VBITS_GE_512-NEXT: cmpne p1.d, p0/z, z0.d, #0 +; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1] +; VBITS_GE_512-NEXT: ld1b { z0.d }, p1/z, [z0.d] ; VBITS_GE_512-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_512-NEXT: uzp1 z0.b, z0.b, z0.b @@ -118,11 +118,11 @@ define void @masked_gather_v16i8(ptr %a, ptr %b) vscale_range(8,0) #0 { ; CHECK-NEXT: ptrue p0.d, vl16 ; CHECK-NEXT: cmeq v0.16b, v0.16b, #0 ; CHECK-NEXT: sunpklo z0.h, z0.b -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] ; CHECK-NEXT: sunpklo z0.s, z0.h ; CHECK-NEXT: sunpklo z0.d, z0.s -; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 -; CHECK-NEXT: ld1b { z0.d }, p0/z, [z1.d] +; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0 +; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1] +; CHECK-NEXT: ld1b { z0.d }, p1/z, [z0.d] ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s ; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: uzp1 z0.b, z0.b, z0.b @@ -194,10 +194,10 @@ define void @masked_gather_v4i16(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: cmeq v0.4h, v0.4h, #0 ; CHECK-NEXT: sunpklo z0.s, z0.h -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] ; CHECK-NEXT: sunpklo z0.d, z0.s -; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 -; CHECK-NEXT: ld1h { z0.d }, p0/z, [z1.d] +; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0 +; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1] +; CHECK-NEXT: ld1h { z0.d }, p1/z, [z0.d] ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s ; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: str d0, [x0] @@ -219,15 +219,15 @@ define void @masked_gather_v8i16(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: cmeq v0.8h, v0.8h, #0 ; VBITS_GE_256-NEXT: sunpklo z1.s, z0.h ; VBITS_GE_256-NEXT: ext v0.16b, v0.16b, v0.16b, #8 -; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1] -; VBITS_GE_256-NEXT: sunpklo z0.s, z0.h ; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s -; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s +; VBITS_GE_256-NEXT: sunpklo z0.s, z0.h ; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z1.d, #0 -; VBITS_GE_256-NEXT: ld1h { z1.d }, p1/z, [z2.d] -; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3] -; VBITS_GE_256-NEXT: cmpne p0.d, p0/z, z0.d, #0 -; VBITS_GE_256-NEXT: ld1h { z0.d }, p0/z, [z2.d] +; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s +; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1] +; VBITS_GE_256-NEXT: ld1h { z1.d }, p1/z, [z1.d] +; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z0.d, #0 +; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1, x8, lsl #3] +; VBITS_GE_256-NEXT: ld1h { z0.d }, p1/z, [z0.d] ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s ; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h ; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s @@ -242,10 +242,10 @@ define void @masked_gather_v8i16(ptr %a, ptr %b) #0 { ; VBITS_GE_512-NEXT: ptrue p0.d, vl8 ; VBITS_GE_512-NEXT: cmeq v0.8h, v0.8h, #0 ; VBITS_GE_512-NEXT: sunpklo z0.s, z0.h -; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1] ; VBITS_GE_512-NEXT: sunpklo z0.d, z0.s -; VBITS_GE_512-NEXT: cmpne p0.d, p0/z, z0.d, #0 -; VBITS_GE_512-NEXT: ld1h { z0.d }, p0/z, [z1.d] +; VBITS_GE_512-NEXT: cmpne p1.d, p0/z, z0.d, #0 +; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1] +; VBITS_GE_512-NEXT: ld1h { z0.d }, p1/z, [z0.d] ; VBITS_GE_512-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_512-NEXT: str q0, [x0] @@ -332,9 +332,9 @@ define void @masked_gather_v4i32(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: cmeq v0.4s, v0.4s, #0 ; CHECK-NEXT: sunpklo z0.d, z0.s -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] -; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 -; CHECK-NEXT: ld1w { z0.d }, p0/z, [z1.d] +; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0 +; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1] +; CHECK-NEXT: ld1w { z0.d }, p1/z, [z0.d] ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s ; CHECK-NEXT: str q0, [x0] ; CHECK-NEXT: ret @@ -354,18 +354,18 @@ define void @masked_gather_v8i32(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: ptrue p2.d, vl4 ; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0] ; VBITS_GE_256-NEXT: ld1d { z1.d }, p2/z, [x1] -; VBITS_GE_256-NEXT: ld1d { z2.d }, p2/z, [x1, x8, lsl #3] ; VBITS_GE_256-NEXT: cmpeq p1.s, p0/z, z0.s, #0 -; VBITS_GE_256-NEXT: punpklo p3.h, p1.b ; VBITS_GE_256-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff +; VBITS_GE_256-NEXT: punpklo p1.h, p1.b +; VBITS_GE_256-NEXT: and p1.b, p1/z, p1.b, p2.b ; VBITS_GE_256-NEXT: ext z0.b, z0.b, z0.b, #16 -; VBITS_GE_256-NEXT: and p1.b, p3/z, p3.b, p2.b -; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s ; VBITS_GE_256-NEXT: ld1w { z1.d }, p1/z, [z1.d] +; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s ; VBITS_GE_256-NEXT: cmpne p1.d, p2/z, z0.d, #0 -; VBITS_GE_256-NEXT: ld1w { z0.d }, p1/z, [z2.d] -; VBITS_GE_256-NEXT: ptrue p1.s, vl4 +; VBITS_GE_256-NEXT: ld1d { z0.d }, p2/z, [x1, x8, lsl #3] ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s +; VBITS_GE_256-NEXT: ld1w { z0.d }, p1/z, [z0.d] +; VBITS_GE_256-NEXT: ptrue p1.s, vl4 ; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_256-NEXT: splice z1.s, p1, z1.s, z0.s ; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0] @@ -460,8 +460,8 @@ define void @masked_gather_v1i64(ptr %a, ptr %b) vscale_range(2,0) #0 { define void @masked_gather_v2i64(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: masked_gather_v2i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldr q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: cmeq v0.2d, v0.2d, #0 ; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 ; CHECK-NEXT: ldr q0, [x1] @@ -481,9 +481,9 @@ define void @masked_gather_v4i64(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] ; CHECK-NEXT: cmpeq p1.d, p0/z, z0.d, #0 -; CHECK-NEXT: ld1d { z0.d }, p1/z, [z1.d] +; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1] +; CHECK-NEXT: ld1d { z0.d }, p1/z, [z0.d] ; CHECK-NEXT: st1d { z0.d }, p0, [x0] ; CHECK-NEXT: ret %cval = load <4 x i64>, ptr %a @@ -500,13 +500,13 @@ define void @masked_gather_v8i64(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 ; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] -; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1, x8, lsl #3] -; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1] -; VBITS_GE_256-NEXT: cmpeq p1.d, p0/z, z0.d, #0 -; VBITS_GE_256-NEXT: ld1d { z0.d }, p1/z, [z1.d] ; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0] +; VBITS_GE_256-NEXT: cmpeq p1.d, p0/z, z0.d, #0 +; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1, x8, lsl #3] +; VBITS_GE_256-NEXT: ld1d { z0.d }, p1/z, [z0.d] ; VBITS_GE_256-NEXT: cmpeq p1.d, p0/z, z1.d, #0 -; VBITS_GE_256-NEXT: ld1d { z1.d }, p1/z, [z2.d] +; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1] +; VBITS_GE_256-NEXT: ld1d { z1.d }, p1/z, [z1.d] ; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3] ; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0] ; VBITS_GE_256-NEXT: ret @@ -515,9 +515,9 @@ define void @masked_gather_v8i64(ptr %a, ptr %b) #0 { ; VBITS_GE_512: // %bb.0: ; VBITS_GE_512-NEXT: ptrue p0.d, vl8 ; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0] -; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1] ; VBITS_GE_512-NEXT: cmpeq p1.d, p0/z, z0.d, #0 -; VBITS_GE_512-NEXT: ld1d { z0.d }, p1/z, [z1.d] +; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1] +; VBITS_GE_512-NEXT: ld1d { z0.d }, p1/z, [z0.d] ; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0] ; VBITS_GE_512-NEXT: ret %cval = load <8 x i64>, ptr %a @@ -533,9 +533,9 @@ define void @masked_gather_v16i64(ptr %a, ptr %b) vscale_range(8,0) #0 { ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.d, vl16 ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] ; CHECK-NEXT: cmpeq p1.d, p0/z, z0.d, #0 -; CHECK-NEXT: ld1d { z0.d }, p1/z, [z1.d] +; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1] +; CHECK-NEXT: ld1d { z0.d }, p1/z, [z0.d] ; CHECK-NEXT: st1d { z0.d }, p0, [x0] ; CHECK-NEXT: ret %cval = load <16 x i64>, ptr %a @@ -551,9 +551,9 @@ define void @masked_gather_v32i64(ptr %a, ptr %b) vscale_range(16,0) #0 { ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.d, vl32 ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] ; CHECK-NEXT: cmpeq p1.d, p0/z, z0.d, #0 -; CHECK-NEXT: ld1d { z0.d }, p1/z, [z1.d] +; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1] +; CHECK-NEXT: ld1d { z0.d }, p1/z, [z0.d] ; CHECK-NEXT: st1d { z0.d }, p0, [x0] ; CHECK-NEXT: ret %cval = load <32 x i64>, ptr %a @@ -603,10 +603,10 @@ define void @masked_gather_v4f16(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: fcmeq v0.4h, v0.4h, #0.0 ; CHECK-NEXT: sunpklo z0.s, z0.h -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] ; CHECK-NEXT: sunpklo z0.d, z0.s -; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 -; CHECK-NEXT: ld1h { z0.d }, p0/z, [z1.d] +; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0 +; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1] +; CHECK-NEXT: ld1h { z0.d }, p1/z, [z0.d] ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s ; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: str d0, [x0] @@ -626,17 +626,17 @@ define void @masked_gather_v8f16(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 ; VBITS_GE_256-NEXT: fcmeq v0.8h, v0.8h, #0.0 -; VBITS_GE_256-NEXT: sunpklo z2.s, z0.h +; VBITS_GE_256-NEXT: sunpklo z1.s, z0.h ; VBITS_GE_256-NEXT: ext v0.16b, v0.16b, v0.16b, #8 -; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1] +; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s ; VBITS_GE_256-NEXT: sunpklo z0.s, z0.h -; VBITS_GE_256-NEXT: sunpklo z2.d, z2.s +; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z1.d, #0 ; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s -; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z2.d, #0 -; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3] +; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1] ; VBITS_GE_256-NEXT: ld1h { z1.d }, p1/z, [z1.d] -; VBITS_GE_256-NEXT: cmpne p0.d, p0/z, z0.d, #0 -; VBITS_GE_256-NEXT: ld1h { z0.d }, p0/z, [z2.d] +; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z0.d, #0 +; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1, x8, lsl #3] +; VBITS_GE_256-NEXT: ld1h { z0.d }, p1/z, [z0.d] ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s ; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h ; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s @@ -651,10 +651,10 @@ define void @masked_gather_v8f16(ptr %a, ptr %b) #0 { ; VBITS_GE_512-NEXT: ptrue p0.d, vl8 ; VBITS_GE_512-NEXT: fcmeq v0.8h, v0.8h, #0.0 ; VBITS_GE_512-NEXT: sunpklo z0.s, z0.h -; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1] ; VBITS_GE_512-NEXT: sunpklo z0.d, z0.s -; VBITS_GE_512-NEXT: cmpne p0.d, p0/z, z0.d, #0 -; VBITS_GE_512-NEXT: ld1h { z0.d }, p0/z, [z1.d] +; VBITS_GE_512-NEXT: cmpne p1.d, p0/z, z0.d, #0 +; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1] +; VBITS_GE_512-NEXT: ld1h { z0.d }, p1/z, [z0.d] ; VBITS_GE_512-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_512-NEXT: str q0, [x0] @@ -741,9 +741,9 @@ define void @masked_gather_v4f32(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: fcmeq v0.4s, v0.4s, #0.0 ; CHECK-NEXT: sunpklo z0.d, z0.s -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] -; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 -; CHECK-NEXT: ld1w { z0.d }, p0/z, [z1.d] +; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0 +; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1] +; CHECK-NEXT: ld1w { z0.d }, p1/z, [z0.d] ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s ; CHECK-NEXT: str q0, [x0] ; CHECK-NEXT: ret @@ -763,18 +763,18 @@ define void @masked_gather_v8f32(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: ptrue p2.d, vl4 ; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0] ; VBITS_GE_256-NEXT: ld1d { z1.d }, p2/z, [x1] -; VBITS_GE_256-NEXT: ld1d { z2.d }, p2/z, [x1, x8, lsl #3] ; VBITS_GE_256-NEXT: fcmeq p1.s, p0/z, z0.s, #0.0 -; VBITS_GE_256-NEXT: punpklo p3.h, p1.b ; VBITS_GE_256-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff +; VBITS_GE_256-NEXT: punpklo p1.h, p1.b +; VBITS_GE_256-NEXT: and p1.b, p1/z, p1.b, p2.b ; VBITS_GE_256-NEXT: ext z0.b, z0.b, z0.b, #16 -; VBITS_GE_256-NEXT: and p1.b, p3/z, p3.b, p2.b -; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s ; VBITS_GE_256-NEXT: ld1w { z1.d }, p1/z, [z1.d] +; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s ; VBITS_GE_256-NEXT: cmpne p1.d, p2/z, z0.d, #0 -; VBITS_GE_256-NEXT: ld1w { z0.d }, p1/z, [z2.d] -; VBITS_GE_256-NEXT: ptrue p1.s, vl4 +; VBITS_GE_256-NEXT: ld1d { z0.d }, p2/z, [x1, x8, lsl #3] ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s +; VBITS_GE_256-NEXT: ld1w { z0.d }, p1/z, [z0.d] +; VBITS_GE_256-NEXT: ptrue p1.s, vl4 ; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_256-NEXT: splice z1.s, p1, z1.s, z0.s ; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0] @@ -869,8 +869,8 @@ define void @masked_gather_v1f64(ptr %a, ptr %b) vscale_range(2,0) #0 { define void @masked_gather_v2f64(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: masked_gather_v2f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldr q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: fcmeq v0.2d, v0.2d, #0.0 ; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 ; CHECK-NEXT: ldr q0, [x1] @@ -1202,8 +1202,8 @@ define void @masked_gather_passthru(ptr %a, ptr %b, ptr %c) vscale_range(16,0) # ; CHECK-NEXT: ld1w { z1.s }, p0/z, [x2] ; CHECK-NEXT: fcmeq p1.s, p0/z, z0.s, #0.0 ; CHECK-NEXT: ld1d { z0.d }, p2/z, [x1] -; CHECK-NEXT: punpklo p3.h, p1.b -; CHECK-NEXT: ld1w { z0.d }, p3/z, [z0.d] +; CHECK-NEXT: punpklo p2.h, p1.b +; CHECK-NEXT: ld1w { z0.d }, p2/z, [z0.d] ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s ; CHECK-NEXT: sel z0.s, p1, z0.s, z1.s ; CHECK-NEXT: st1w { z0.s }, p0, [x0] diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-loads.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-loads.ll index 467378e7da59..c22d9e71c51a 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-loads.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-loads.ll @@ -35,9 +35,9 @@ define <2 x half> @masked_load_v2f16(ptr %ap, ptr %bp) vscale_range(2,0) #0 { define <2 x float> @masked_load_v2f32(ptr %ap, ptr %bp) vscale_range(2,0) #0 { ; CHECK-LABEL: masked_load_v2f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: ldr d0, [x0] ; CHECK-NEXT: ldr d1, [x1] +; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: fcmeq v0.2s, v0.2s, v1.2s ; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0 ; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0] @@ -53,9 +53,9 @@ define <2 x float> @masked_load_v2f32(ptr %ap, ptr %bp) vscale_range(2,0) #0 { define <4 x float> @masked_load_v4f32(ptr %ap, ptr %bp) vscale_range(1,0) #0 { ; CHECK-LABEL: masked_load_v4f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldr q0, [x0] ; CHECK-NEXT: ldr q1, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: fcmeq v0.4s, v0.4s, v1.4s ; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0 ; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0] @@ -401,8 +401,8 @@ define void @masked_load_sext_v32i8i16(ptr %ap, ptr %bp, ptr %c) #0 { define void @masked_load_sext_v16i8i32(ptr %ap, ptr %bp, ptr %c) #0 { ; VBITS_GE_256-LABEL: masked_load_sext_v16i8i32: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.b, vl16 ; VBITS_GE_256-NEXT: ldr q0, [x1] +; VBITS_GE_256-NEXT: ptrue p0.b, vl16 ; VBITS_GE_256-NEXT: mov x8, #8 // =0x8 ; VBITS_GE_256-NEXT: cmeq v0.16b, v0.16b, #0 ; VBITS_GE_256-NEXT: cmpne p0.b, p0/z, z0.b, #0 @@ -436,8 +436,8 @@ define void @masked_load_sext_v16i8i32(ptr %ap, ptr %bp, ptr %c) #0 { define void @masked_load_sext_v8i8i64(ptr %ap, ptr %bp, ptr %c) #0 { ; VBITS_GE_256-LABEL: masked_load_sext_v8i8i64: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.b, vl8 ; VBITS_GE_256-NEXT: ldr d0, [x1] +; VBITS_GE_256-NEXT: ptrue p0.b, vl8 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 ; VBITS_GE_256-NEXT: cmeq v0.8b, v0.8b, #0 ; VBITS_GE_256-NEXT: cmpne p0.b, p0/z, z0.b, #0 @@ -504,8 +504,8 @@ define void @masked_load_sext_v16i16i32(ptr %ap, ptr %bp, ptr %c) #0 { define void @masked_load_sext_v8i16i64(ptr %ap, ptr %bp, ptr %c) #0 { ; VBITS_GE_256-LABEL: masked_load_sext_v8i16i64: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.h, vl8 ; VBITS_GE_256-NEXT: ldr q0, [x1] +; VBITS_GE_256-NEXT: ptrue p0.h, vl8 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 ; VBITS_GE_256-NEXT: cmeq v0.8h, v0.8h, #0 ; VBITS_GE_256-NEXT: cmpne p0.h, p0/z, z0.h, #0 @@ -603,8 +603,8 @@ define void @masked_load_zext_v32i8i16(ptr %ap, ptr %bp, ptr %c) #0 { define void @masked_load_zext_v16i8i32(ptr %ap, ptr %bp, ptr %c) #0 { ; VBITS_GE_256-LABEL: masked_load_zext_v16i8i32: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.b, vl16 ; VBITS_GE_256-NEXT: ldr q0, [x1] +; VBITS_GE_256-NEXT: ptrue p0.b, vl16 ; VBITS_GE_256-NEXT: mov x8, #8 // =0x8 ; VBITS_GE_256-NEXT: cmeq v0.16b, v0.16b, #0 ; VBITS_GE_256-NEXT: cmpne p0.b, p0/z, z0.b, #0 @@ -638,8 +638,8 @@ define void @masked_load_zext_v16i8i32(ptr %ap, ptr %bp, ptr %c) #0 { define void @masked_load_zext_v8i8i64(ptr %ap, ptr %bp, ptr %c) #0 { ; VBITS_GE_256-LABEL: masked_load_zext_v8i8i64: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.b, vl8 ; VBITS_GE_256-NEXT: ldr d0, [x1] +; VBITS_GE_256-NEXT: ptrue p0.b, vl8 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 ; VBITS_GE_256-NEXT: cmeq v0.8b, v0.8b, #0 ; VBITS_GE_256-NEXT: cmpne p0.b, p0/z, z0.b, #0 @@ -706,8 +706,8 @@ define void @masked_load_zext_v16i16i32(ptr %ap, ptr %bp, ptr %c) #0 { define void @masked_load_zext_v8i16i64(ptr %ap, ptr %bp, ptr %c) #0 { ; VBITS_GE_256-LABEL: masked_load_zext_v8i16i64: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.h, vl8 ; VBITS_GE_256-NEXT: ldr q0, [x1] +; VBITS_GE_256-NEXT: ptrue p0.h, vl8 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 ; VBITS_GE_256-NEXT: cmeq v0.8h, v0.8h, #0 ; VBITS_GE_256-NEXT: cmpne p0.h, p0/z, z0.h, #0 @@ -782,11 +782,11 @@ define void @masked_load_sext_v32i8i16_m16(ptr %ap, ptr %bp, ptr %c) #0 { ; VBITS_GE_256-NEXT: mov z0.h, p1/z, #-1 // =0xffffffffffffffff ; VBITS_GE_256-NEXT: mov z1.h, p2/z, #-1 // =0xffffffffffffffff ; VBITS_GE_256-NEXT: ptrue p1.b, vl16 -; VBITS_GE_256-NEXT: ptrue p2.b, vl32 ; VBITS_GE_256-NEXT: uzp1 z0.b, z0.b, z0.b ; VBITS_GE_256-NEXT: uzp1 z1.b, z1.b, z1.b ; VBITS_GE_256-NEXT: splice z1.b, p1, z1.b, z0.b -; VBITS_GE_256-NEXT: cmpne p1.b, p2/z, z1.b, #0 +; VBITS_GE_256-NEXT: ptrue p1.b, vl32 +; VBITS_GE_256-NEXT: cmpne p1.b, p1/z, z1.b, #0 ; VBITS_GE_256-NEXT: ld1b { z0.b }, p1/z, [x0] ; VBITS_GE_256-NEXT: sunpklo z1.h, z0.b ; VBITS_GE_256-NEXT: ext z0.b, z0.b, z0.b, #16 @@ -1000,11 +1000,11 @@ define void @masked_load_sext_v8i32i64_m64(ptr %ap, ptr %bp, ptr %c) #0 { ; VBITS_GE_256-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff ; VBITS_GE_256-NEXT: mov z1.d, p2/z, #-1 // =0xffffffffffffffff ; VBITS_GE_256-NEXT: ptrue p1.s, vl4 -; VBITS_GE_256-NEXT: ptrue p2.s, vl8 ; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s ; VBITS_GE_256-NEXT: splice z1.s, p1, z1.s, z0.s -; VBITS_GE_256-NEXT: cmpne p1.s, p2/z, z1.s, #0 +; VBITS_GE_256-NEXT: ptrue p1.s, vl8 +; VBITS_GE_256-NEXT: cmpne p1.s, p1/z, z1.s, #0 ; VBITS_GE_256-NEXT: ld1w { z0.s }, p1/z, [x0] ; VBITS_GE_256-NEXT: sunpklo z1.d, z0.s ; VBITS_GE_256-NEXT: ext z0.b, z0.b, z0.b, #16 @@ -1041,11 +1041,11 @@ define void @masked_load_zext_v32i8i16_m16(ptr %ap, ptr %bp, ptr %c) #0 { ; VBITS_GE_256-NEXT: mov z0.h, p1/z, #-1 // =0xffffffffffffffff ; VBITS_GE_256-NEXT: mov z1.h, p2/z, #-1 // =0xffffffffffffffff ; VBITS_GE_256-NEXT: ptrue p1.b, vl16 -; VBITS_GE_256-NEXT: ptrue p2.b, vl32 ; VBITS_GE_256-NEXT: uzp1 z0.b, z0.b, z0.b ; VBITS_GE_256-NEXT: uzp1 z1.b, z1.b, z1.b ; VBITS_GE_256-NEXT: splice z1.b, p1, z1.b, z0.b -; VBITS_GE_256-NEXT: cmpne p1.b, p2/z, z1.b, #0 +; VBITS_GE_256-NEXT: ptrue p1.b, vl32 +; VBITS_GE_256-NEXT: cmpne p1.b, p1/z, z1.b, #0 ; VBITS_GE_256-NEXT: ld1b { z0.b }, p1/z, [x0] ; VBITS_GE_256-NEXT: uunpklo z1.h, z0.b ; VBITS_GE_256-NEXT: ext z0.b, z0.b, z0.b, #16 @@ -1259,11 +1259,11 @@ define void @masked_load_zext_v8i32i64_m64(ptr %ap, ptr %bp, ptr %c) #0 { ; VBITS_GE_256-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff ; VBITS_GE_256-NEXT: mov z1.d, p2/z, #-1 // =0xffffffffffffffff ; VBITS_GE_256-NEXT: ptrue p1.s, vl4 -; VBITS_GE_256-NEXT: ptrue p2.s, vl8 ; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s ; VBITS_GE_256-NEXT: splice z1.s, p1, z1.s, z0.s -; VBITS_GE_256-NEXT: cmpne p1.s, p2/z, z1.s, #0 +; VBITS_GE_256-NEXT: ptrue p1.s, vl8 +; VBITS_GE_256-NEXT: cmpne p1.s, p1/z, z1.s, #0 ; VBITS_GE_256-NEXT: ld1w { z0.s }, p1/z, [x0] ; VBITS_GE_256-NEXT: uunpklo z1.d, z0.s ; VBITS_GE_256-NEXT: ext z0.b, z0.b, z0.b, #16 diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-scatter.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-scatter.ll index e2d341c22efc..e3e06dcdf17f 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-scatter.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-scatter.ll @@ -39,12 +39,12 @@ define void @masked_scatter_v4i8(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-NEXT: ushll v0.8h, v0.8b, #0 ; CHECK-NEXT: cmeq v1.4h, v0.4h, #0 ; CHECK-NEXT: uunpklo z0.s, z0.h -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1] ; CHECK-NEXT: sunpklo z1.s, z1.h ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: sunpklo z1.d, z1.s -; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0 -; CHECK-NEXT: st1b { z0.d }, p0, [z2.d] +; CHECK-NEXT: cmpne p1.d, p0/z, z1.d, #0 +; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] +; CHECK-NEXT: st1b { z0.d }, p1, [z1.d] ; CHECK-NEXT: ret %vals = load <4 x i8>, ptr %a %ptrs = load <4 x ptr>, ptr %b @@ -61,11 +61,11 @@ define void @masked_scatter_v8i8(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 ; VBITS_GE_256-NEXT: cmeq v1.8b, v0.8b, #0 ; VBITS_GE_256-NEXT: zip1 v3.8b, v0.8b, v0.8b +; VBITS_GE_256-NEXT: ld1d { z4.d }, p0/z, [x1, x8, lsl #3] ; VBITS_GE_256-NEXT: zip1 v2.8b, v1.8b, v0.8b ; VBITS_GE_256-NEXT: zip2 v1.8b, v1.8b, v0.8b ; VBITS_GE_256-NEXT: zip2 v0.8b, v0.8b, v0.8b ; VBITS_GE_256-NEXT: uunpklo z3.s, z3.h -; VBITS_GE_256-NEXT: ld1d { z4.d }, p0/z, [x1] ; VBITS_GE_256-NEXT: shl v2.4h, v2.4h, #8 ; VBITS_GE_256-NEXT: shl v1.4h, v1.4h, #8 ; VBITS_GE_256-NEXT: uunpklo z0.s, z0.h @@ -78,10 +78,10 @@ define void @masked_scatter_v8i8(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: sunpklo z2.d, z2.s ; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s ; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z2.d, #0 -; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3] +; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1] ; VBITS_GE_256-NEXT: cmpne p0.d, p0/z, z1.d, #0 -; VBITS_GE_256-NEXT: st1b { z3.d }, p1, [z4.d] -; VBITS_GE_256-NEXT: st1b { z0.d }, p0, [z2.d] +; VBITS_GE_256-NEXT: st1b { z3.d }, p1, [z2.d] +; VBITS_GE_256-NEXT: st1b { z0.d }, p0, [z4.d] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: masked_scatter_v8i8: @@ -92,12 +92,12 @@ define void @masked_scatter_v8i8(ptr %a, ptr %b) #0 { ; VBITS_GE_512-NEXT: uunpklo z0.h, z0.b ; VBITS_GE_512-NEXT: sunpklo z1.h, z1.b ; VBITS_GE_512-NEXT: uunpklo z0.s, z0.h -; VBITS_GE_512-NEXT: ld1d { z2.d }, p0/z, [x1] ; VBITS_GE_512-NEXT: sunpklo z1.s, z1.h ; VBITS_GE_512-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_512-NEXT: sunpklo z1.d, z1.s -; VBITS_GE_512-NEXT: cmpne p0.d, p0/z, z1.d, #0 -; VBITS_GE_512-NEXT: st1b { z0.d }, p0, [z2.d] +; VBITS_GE_512-NEXT: cmpne p1.d, p0/z, z1.d, #0 +; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1] +; VBITS_GE_512-NEXT: st1b { z0.d }, p1, [z1.d] ; VBITS_GE_512-NEXT: ret %vals = load <8 x i8>, ptr %a %ptrs = load <8 x ptr>, ptr %b @@ -115,12 +115,12 @@ define void @masked_scatter_v16i8(ptr %a, ptr %b) vscale_range(8,0) #0 { ; CHECK-NEXT: uunpklo z0.h, z0.b ; CHECK-NEXT: sunpklo z1.h, z1.b ; CHECK-NEXT: uunpklo z0.s, z0.h -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1] ; CHECK-NEXT: sunpklo z1.s, z1.h ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: sunpklo z1.d, z1.s -; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0 -; CHECK-NEXT: st1b { z0.d }, p0, [z2.d] +; CHECK-NEXT: cmpne p1.d, p0/z, z1.d, #0 +; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] +; CHECK-NEXT: st1b { z0.d }, p1, [z1.d] ; CHECK-NEXT: ret %vals = load <16 x i8>, ptr %a %ptrs = load <16 x ptr>, ptr %b @@ -135,13 +135,13 @@ define void @masked_scatter_v32i8(ptr %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-NEXT: ptrue p0.b, vl32 ; CHECK-NEXT: ptrue p1.d, vl32 ; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0] -; CHECK-NEXT: ld1d { z1.d }, p1/z, [x1] +; CHECK-NEXT: uunpklo z1.h, z0.b ; CHECK-NEXT: cmpeq p0.b, p0/z, z0.b, #0 -; CHECK-NEXT: uunpklo z0.h, z0.b +; CHECK-NEXT: uunpklo z0.s, z1.h ; CHECK-NEXT: punpklo p0.h, p0.b -; CHECK-NEXT: uunpklo z0.s, z0.h -; CHECK-NEXT: uunpklo z0.d, z0.s +; CHECK-NEXT: ld1d { z1.d }, p1/z, [x1] ; CHECK-NEXT: punpklo p0.h, p0.b +; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: punpklo p0.h, p0.b ; CHECK-NEXT: st1b { z0.d }, p0, [z1.d] ; CHECK-NEXT: ret @@ -187,10 +187,10 @@ define void @masked_scatter_v4i16(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: sunpklo z1.s, z1.h ; CHECK-NEXT: uunpklo z0.d, z0.s -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1] ; CHECK-NEXT: sunpklo z1.d, z1.s -; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0 -; CHECK-NEXT: st1h { z0.d }, p0, [z2.d] +; CHECK-NEXT: cmpne p1.d, p0/z, z1.d, #0 +; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] +; CHECK-NEXT: st1h { z0.d }, p1, [z1.d] ; CHECK-NEXT: ret %vals = load <4 x i16>, ptr %a %ptrs = load <4 x ptr>, ptr %b @@ -208,20 +208,20 @@ define void @masked_scatter_v8i16(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: cmeq v1.8h, v0.8h, #0 ; VBITS_GE_256-NEXT: uunpklo z3.s, z0.h ; VBITS_GE_256-NEXT: ext v0.16b, v0.16b, v0.16b, #8 +; VBITS_GE_256-NEXT: ld1d { z4.d }, p0/z, [x1, x8, lsl #3] ; VBITS_GE_256-NEXT: sunpklo z2.s, z1.h ; VBITS_GE_256-NEXT: ext v1.16b, v1.16b, v1.16b, #8 ; VBITS_GE_256-NEXT: uunpklo z0.s, z0.h ; VBITS_GE_256-NEXT: uunpklo z3.d, z3.s -; VBITS_GE_256-NEXT: ld1d { z4.d }, p0/z, [x1] ; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h ; VBITS_GE_256-NEXT: sunpklo z2.d, z2.s ; VBITS_GE_256-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s ; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z2.d, #0 -; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3] -; VBITS_GE_256-NEXT: st1h { z3.d }, p1, [z4.d] +; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1] ; VBITS_GE_256-NEXT: cmpne p0.d, p0/z, z1.d, #0 -; VBITS_GE_256-NEXT: st1h { z0.d }, p0, [z2.d] +; VBITS_GE_256-NEXT: st1h { z3.d }, p1, [z2.d] +; VBITS_GE_256-NEXT: st1h { z0.d }, p0, [z4.d] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: masked_scatter_v8i16: @@ -232,10 +232,10 @@ define void @masked_scatter_v8i16(ptr %a, ptr %b) #0 { ; VBITS_GE_512-NEXT: uunpklo z0.s, z0.h ; VBITS_GE_512-NEXT: sunpklo z1.s, z1.h ; VBITS_GE_512-NEXT: uunpklo z0.d, z0.s -; VBITS_GE_512-NEXT: ld1d { z2.d }, p0/z, [x1] ; VBITS_GE_512-NEXT: sunpklo z1.d, z1.s -; VBITS_GE_512-NEXT: cmpne p0.d, p0/z, z1.d, #0 -; VBITS_GE_512-NEXT: st1h { z0.d }, p0, [z2.d] +; VBITS_GE_512-NEXT: cmpne p1.d, p0/z, z1.d, #0 +; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1] +; VBITS_GE_512-NEXT: st1h { z0.d }, p1, [z1.d] ; VBITS_GE_512-NEXT: ret %vals = load <8 x i16>, ptr %a %ptrs = load <8 x ptr>, ptr %b @@ -253,9 +253,9 @@ define void @masked_scatter_v16i16(ptr %a, ptr %b) vscale_range(8,0) #0 { ; CHECK-NEXT: ld1d { z1.d }, p1/z, [x1] ; CHECK-NEXT: cmpeq p0.h, p0/z, z0.h, #0 ; CHECK-NEXT: uunpklo z0.s, z0.h -; CHECK-NEXT: punpklo p0.h, p0.b ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: punpklo p0.h, p0.b +; CHECK-NEXT: punpklo p0.h, p0.b ; CHECK-NEXT: st1h { z0.d }, p0, [z1.d] ; CHECK-NEXT: ret %vals = load <16 x i16>, ptr %a @@ -274,9 +274,9 @@ define void @masked_scatter_v32i16(ptr %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-NEXT: ld1d { z1.d }, p1/z, [x1] ; CHECK-NEXT: cmpeq p0.h, p0/z, z0.h, #0 ; CHECK-NEXT: uunpklo z0.s, z0.h -; CHECK-NEXT: punpklo p0.h, p0.b ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: punpklo p0.h, p0.b +; CHECK-NEXT: punpklo p0.h, p0.b ; CHECK-NEXT: st1h { z0.d }, p0, [z1.d] ; CHECK-NEXT: ret %vals = load <32 x i16>, ptr %a @@ -317,9 +317,9 @@ define void @masked_scatter_v4i32(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-NEXT: cmeq v1.4s, v0.4s, #0 ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: sunpklo z1.d, z1.s -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1] -; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0 -; CHECK-NEXT: st1w { z0.d }, p0, [z2.d] +; CHECK-NEXT: cmpne p1.d, p0/z, z1.d, #0 +; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] +; CHECK-NEXT: st1w { z0.d }, p1, [z1.d] ; CHECK-NEXT: ret %vals = load <4 x i32>, ptr %a %ptrs = load <4 x ptr>, ptr %b @@ -333,21 +333,21 @@ define void @masked_scatter_v8i32(ptr %a, ptr %b) #0 { ; VBITS_GE_256: // %bb.0: ; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 +; VBITS_GE_256-NEXT: ptrue p1.d, vl4 ; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0] -; VBITS_GE_256-NEXT: cmpeq p1.s, p0/z, z0.s, #0 -; VBITS_GE_256-NEXT: ptrue p0.d, vl4 +; VBITS_GE_256-NEXT: ld1d { z3.d }, p1/z, [x1] +; VBITS_GE_256-NEXT: ld1d { z4.d }, p1/z, [x1, x8, lsl #3] +; VBITS_GE_256-NEXT: cmpeq p0.s, p0/z, z0.s, #0 ; VBITS_GE_256-NEXT: uunpklo z2.d, z0.s ; VBITS_GE_256-NEXT: ext z0.b, z0.b, z0.b, #16 -; VBITS_GE_256-NEXT: mov z1.s, p1/z, #-1 // =0xffffffffffffffff -; VBITS_GE_256-NEXT: punpklo p2.h, p1.b ; VBITS_GE_256-NEXT: uunpklo z0.d, z0.s +; VBITS_GE_256-NEXT: mov z1.s, p0/z, #-1 // =0xffffffffffffffff +; VBITS_GE_256-NEXT: punpklo p0.h, p0.b +; VBITS_GE_256-NEXT: and p0.b, p0/z, p0.b, p1.b ; VBITS_GE_256-NEXT: ext z1.b, z1.b, z1.b, #16 -; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x1] -; VBITS_GE_256-NEXT: ld1d { z4.d }, p0/z, [x1, x8, lsl #3] +; VBITS_GE_256-NEXT: st1w { z2.d }, p0, [z3.d] ; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s -; VBITS_GE_256-NEXT: and p1.b, p2/z, p2.b, p0.b -; VBITS_GE_256-NEXT: cmpne p0.d, p0/z, z1.d, #0 -; VBITS_GE_256-NEXT: st1w { z2.d }, p1, [z3.d] +; VBITS_GE_256-NEXT: cmpne p0.d, p1/z, z1.d, #0 ; VBITS_GE_256-NEXT: st1w { z0.d }, p0, [z4.d] ; VBITS_GE_256-NEXT: ret ; @@ -434,8 +434,8 @@ define void @masked_scatter_v1i64(ptr %a, ptr %b) vscale_range(2,0) #0 { define void @masked_scatter_v2i64(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: masked_scatter_v2i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldr q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: cmeq v1.2d, v0.2d, #0 ; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0 ; CHECK-NEXT: ldr q1, [x1] @@ -454,8 +454,8 @@ define void @masked_scatter_v4i64(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] ; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] -; CHECK-NEXT: cmpeq p0.d, p0/z, z0.d, #0 -; CHECK-NEXT: st1d { z0.d }, p0, [z1.d] +; CHECK-NEXT: cmpeq p1.d, p0/z, z0.d, #0 +; CHECK-NEXT: st1d { z0.d }, p1, [z1.d] ; CHECK-NEXT: ret %vals = load <4 x i64>, ptr %a %ptrs = load <4 x ptr>, ptr %b @@ -484,8 +484,8 @@ define void @masked_scatter_v8i64(ptr %a, ptr %b) #0 { ; VBITS_GE_512-NEXT: ptrue p0.d, vl8 ; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0] ; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1] -; VBITS_GE_512-NEXT: cmpeq p0.d, p0/z, z0.d, #0 -; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [z1.d] +; VBITS_GE_512-NEXT: cmpeq p1.d, p0/z, z0.d, #0 +; VBITS_GE_512-NEXT: st1d { z0.d }, p1, [z1.d] ; VBITS_GE_512-NEXT: ret %vals = load <8 x i64>, ptr %a %ptrs = load <8 x ptr>, ptr %b @@ -500,8 +500,8 @@ define void @masked_scatter_v16i64(ptr %a, ptr %b) vscale_range(8,0) #0 { ; CHECK-NEXT: ptrue p0.d, vl16 ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] ; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] -; CHECK-NEXT: cmpeq p0.d, p0/z, z0.d, #0 -; CHECK-NEXT: st1d { z0.d }, p0, [z1.d] +; CHECK-NEXT: cmpeq p1.d, p0/z, z0.d, #0 +; CHECK-NEXT: st1d { z0.d }, p1, [z1.d] ; CHECK-NEXT: ret %vals = load <16 x i64>, ptr %a %ptrs = load <16 x ptr>, ptr %b @@ -516,8 +516,8 @@ define void @masked_scatter_v32i64(ptr %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-NEXT: ptrue p0.d, vl32 ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] ; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] -; CHECK-NEXT: cmpeq p0.d, p0/z, z0.d, #0 -; CHECK-NEXT: st1d { z0.d }, p0, [z1.d] +; CHECK-NEXT: cmpeq p1.d, p0/z, z0.d, #0 +; CHECK-NEXT: st1d { z0.d }, p1, [z1.d] ; CHECK-NEXT: ret %vals = load <32 x i64>, ptr %a %ptrs = load <32 x ptr>, ptr %b @@ -539,15 +539,15 @@ define void @masked_scatter_v2f16(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-NEXT: fcmeq v2.4h, v1.4h, #0.0 ; CHECK-NEXT: uunpklo z1.s, z1.h ; CHECK-NEXT: sshll v2.4s, v2.4h, #0 -; CHECK-NEXT: uunpklo z1.d, z1.s ; CHECK-NEXT: mov v0.h[0], v2.h[0] ; CHECK-NEXT: mov w8, v2.s[1] ; CHECK-NEXT: mov v0.h[1], w8 ; CHECK-NEXT: sunpklo z0.s, z0.h ; CHECK-NEXT: sunpklo z0.d, z0.s ; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 -; CHECK-NEXT: ldr q0, [x1] -; CHECK-NEXT: st1h { z1.d }, p0, [z0.d] +; CHECK-NEXT: uunpklo z0.d, z1.s +; CHECK-NEXT: ldr q1, [x1] +; CHECK-NEXT: st1h { z0.d }, p0, [z1.d] ; CHECK-NEXT: ret %vals = load <2 x half>, ptr %a %ptrs = load <2 x ptr>, ptr %b @@ -565,10 +565,10 @@ define void @masked_scatter_v4f16(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: sunpklo z1.s, z1.h ; CHECK-NEXT: uunpklo z0.d, z0.s -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1] ; CHECK-NEXT: sunpklo z1.d, z1.s -; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0 -; CHECK-NEXT: st1h { z0.d }, p0, [z2.d] +; CHECK-NEXT: cmpne p1.d, p0/z, z1.d, #0 +; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] +; CHECK-NEXT: st1h { z0.d }, p1, [z1.d] ; CHECK-NEXT: ret %vals = load <4 x half>, ptr %a %ptrs = load <4 x ptr>, ptr %b @@ -586,20 +586,20 @@ define void @masked_scatter_v8f16(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: fcmeq v1.8h, v0.8h, #0.0 ; VBITS_GE_256-NEXT: uunpklo z3.s, z0.h ; VBITS_GE_256-NEXT: ext v0.16b, v0.16b, v0.16b, #8 +; VBITS_GE_256-NEXT: ld1d { z4.d }, p0/z, [x1, x8, lsl #3] ; VBITS_GE_256-NEXT: sunpklo z2.s, z1.h ; VBITS_GE_256-NEXT: ext v1.16b, v1.16b, v1.16b, #8 ; VBITS_GE_256-NEXT: uunpklo z0.s, z0.h ; VBITS_GE_256-NEXT: uunpklo z3.d, z3.s -; VBITS_GE_256-NEXT: ld1d { z4.d }, p0/z, [x1] ; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h ; VBITS_GE_256-NEXT: sunpklo z2.d, z2.s ; VBITS_GE_256-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s ; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z2.d, #0 -; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3] -; VBITS_GE_256-NEXT: st1h { z3.d }, p1, [z4.d] +; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1] ; VBITS_GE_256-NEXT: cmpne p0.d, p0/z, z1.d, #0 -; VBITS_GE_256-NEXT: st1h { z0.d }, p0, [z2.d] +; VBITS_GE_256-NEXT: st1h { z3.d }, p1, [z2.d] +; VBITS_GE_256-NEXT: st1h { z0.d }, p0, [z4.d] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: masked_scatter_v8f16: @@ -610,10 +610,10 @@ define void @masked_scatter_v8f16(ptr %a, ptr %b) #0 { ; VBITS_GE_512-NEXT: uunpklo z0.s, z0.h ; VBITS_GE_512-NEXT: sunpklo z1.s, z1.h ; VBITS_GE_512-NEXT: uunpklo z0.d, z0.s -; VBITS_GE_512-NEXT: ld1d { z2.d }, p0/z, [x1] ; VBITS_GE_512-NEXT: sunpklo z1.d, z1.s -; VBITS_GE_512-NEXT: cmpne p0.d, p0/z, z1.d, #0 -; VBITS_GE_512-NEXT: st1h { z0.d }, p0, [z2.d] +; VBITS_GE_512-NEXT: cmpne p1.d, p0/z, z1.d, #0 +; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1] +; VBITS_GE_512-NEXT: st1h { z0.d }, p1, [z1.d] ; VBITS_GE_512-NEXT: ret %vals = load <8 x half>, ptr %a %ptrs = load <8 x ptr>, ptr %b @@ -631,9 +631,9 @@ define void @masked_scatter_v16f16(ptr %a, ptr %b) vscale_range(8,0) #0 { ; CHECK-NEXT: ld1d { z1.d }, p1/z, [x1] ; CHECK-NEXT: fcmeq p0.h, p0/z, z0.h, #0.0 ; CHECK-NEXT: uunpklo z0.s, z0.h -; CHECK-NEXT: punpklo p0.h, p0.b ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: punpklo p0.h, p0.b +; CHECK-NEXT: punpklo p0.h, p0.b ; CHECK-NEXT: st1h { z0.d }, p0, [z1.d] ; CHECK-NEXT: ret %vals = load <16 x half>, ptr %a @@ -652,9 +652,9 @@ define void @masked_scatter_v32f16(ptr %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-NEXT: ld1d { z1.d }, p1/z, [x1] ; CHECK-NEXT: fcmeq p0.h, p0/z, z0.h, #0.0 ; CHECK-NEXT: uunpklo z0.s, z0.h -; CHECK-NEXT: punpklo p0.h, p0.b ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: punpklo p0.h, p0.b +; CHECK-NEXT: punpklo p0.h, p0.b ; CHECK-NEXT: st1h { z0.d }, p0, [z1.d] ; CHECK-NEXT: ret %vals = load <32 x half>, ptr %a @@ -695,9 +695,9 @@ define void @masked_scatter_v4f32(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-NEXT: fcmeq v1.4s, v0.4s, #0.0 ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: sunpklo z1.d, z1.s -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1] -; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0 -; CHECK-NEXT: st1w { z0.d }, p0, [z2.d] +; CHECK-NEXT: cmpne p1.d, p0/z, z1.d, #0 +; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] +; CHECK-NEXT: st1w { z0.d }, p1, [z1.d] ; CHECK-NEXT: ret %vals = load <4 x float>, ptr %a %ptrs = load <4 x ptr>, ptr %b @@ -711,21 +711,21 @@ define void @masked_scatter_v8f32(ptr %a, ptr %b) #0 { ; VBITS_GE_256: // %bb.0: ; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 +; VBITS_GE_256-NEXT: ptrue p1.d, vl4 ; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0] -; VBITS_GE_256-NEXT: fcmeq p1.s, p0/z, z0.s, #0.0 -; VBITS_GE_256-NEXT: ptrue p0.d, vl4 +; VBITS_GE_256-NEXT: ld1d { z3.d }, p1/z, [x1] +; VBITS_GE_256-NEXT: ld1d { z4.d }, p1/z, [x1, x8, lsl #3] +; VBITS_GE_256-NEXT: fcmeq p0.s, p0/z, z0.s, #0.0 ; VBITS_GE_256-NEXT: uunpklo z2.d, z0.s ; VBITS_GE_256-NEXT: ext z0.b, z0.b, z0.b, #16 -; VBITS_GE_256-NEXT: mov z1.s, p1/z, #-1 // =0xffffffffffffffff -; VBITS_GE_256-NEXT: punpklo p2.h, p1.b ; VBITS_GE_256-NEXT: uunpklo z0.d, z0.s -; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x1] -; VBITS_GE_256-NEXT: ld1d { z4.d }, p0/z, [x1, x8, lsl #3] +; VBITS_GE_256-NEXT: mov z1.s, p0/z, #-1 // =0xffffffffffffffff +; VBITS_GE_256-NEXT: punpklo p0.h, p0.b +; VBITS_GE_256-NEXT: and p0.b, p0/z, p0.b, p1.b ; VBITS_GE_256-NEXT: ext z1.b, z1.b, z1.b, #16 +; VBITS_GE_256-NEXT: st1w { z2.d }, p0, [z3.d] ; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s -; VBITS_GE_256-NEXT: and p1.b, p2/z, p2.b, p0.b -; VBITS_GE_256-NEXT: cmpne p0.d, p0/z, z1.d, #0 -; VBITS_GE_256-NEXT: st1w { z2.d }, p1, [z3.d] +; VBITS_GE_256-NEXT: cmpne p0.d, p1/z, z1.d, #0 ; VBITS_GE_256-NEXT: st1w { z0.d }, p0, [z4.d] ; VBITS_GE_256-NEXT: ret ; @@ -812,8 +812,8 @@ define void @masked_scatter_v1f64(ptr %a, ptr %b) vscale_range(8,0) #0 { define void @masked_scatter_v2f64(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: masked_scatter_v2f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldr q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: fcmeq v1.2d, v0.2d, #0.0 ; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0 ; CHECK-NEXT: ldr q1, [x1] diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-stores.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-stores.ll index 68fb4cc6afb0..b0d4f79aea11 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-stores.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-stores.ll @@ -34,9 +34,9 @@ define void @masked_store_v2f16(ptr %ap, ptr %bp) vscale_range(2,0) #0 { define void @masked_store_v2f32(ptr %ap, ptr %bp) vscale_range(2,0) #0 { ; CHECK-LABEL: masked_store_v2f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: ldr d0, [x0] ; CHECK-NEXT: ldr d1, [x1] +; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: fcmeq v1.2s, v0.2s, v1.2s ; CHECK-NEXT: cmpne p0.s, p0/z, z1.s, #0 ; CHECK-NEXT: st1w { z0.s }, p0, [x1] @@ -51,9 +51,9 @@ define void @masked_store_v2f32(ptr %ap, ptr %bp) vscale_range(2,0) #0 { define void @masked_store_v4f32(ptr %ap, ptr %bp) vscale_range(1,0) #0 { ; CHECK-LABEL: masked_store_v4f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldr q0, [x0] ; CHECK-NEXT: ldr q1, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: fcmeq v1.4s, v0.4s, v1.4s ; CHECK-NEXT: cmpne p0.s, p0/z, z1.s, #0 ; CHECK-NEXT: st1w { z0.s }, p0, [x1] @@ -161,11 +161,11 @@ define void @masked_store_trunc_v8i64i8(ptr %ap, ptr %bp, ptr %dest) #0 { ; VBITS_GE_256-NEXT: mov z3.d, p0/z, #-1 // =0xffffffffffffffff ; VBITS_GE_256-NEXT: ptrue p0.s, vl4 ; VBITS_GE_256-NEXT: uzp1 z2.s, z2.s, z2.s -; VBITS_GE_256-NEXT: uzp1 z3.s, z3.s, z3.s ; VBITS_GE_256-NEXT: splice z1.s, p0, z1.s, z0.s +; VBITS_GE_256-NEXT: uzp1 z3.s, z3.s, z3.s ; VBITS_GE_256-NEXT: splice z3.s, p0, z3.s, z2.s -; VBITS_GE_256-NEXT: cmpne p0.s, p1/z, z3.s, #0 -; VBITS_GE_256-NEXT: st1b { z1.s }, p0, [x2] +; VBITS_GE_256-NEXT: cmpne p1.s, p1/z, z3.s, #0 +; VBITS_GE_256-NEXT: st1b { z1.s }, p1, [x2] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: masked_store_trunc_v8i64i8: @@ -197,14 +197,14 @@ define void @masked_store_trunc_v8i64i16(ptr %ap, ptr %bp, ptr %dest) #0 { ; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_256-NEXT: cmpeq p0.d, p0/z, z1.d, z3.d ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s +; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_256-NEXT: mov z2.d, p1/z, #-1 // =0xffffffffffffffff +; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h ; VBITS_GE_256-NEXT: mov z3.d, p0/z, #-1 // =0xffffffffffffffff ; VBITS_GE_256-NEXT: ptrue p0.s, vl4 -; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h -; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h ; VBITS_GE_256-NEXT: uzp1 z2.s, z2.s, z2.s -; VBITS_GE_256-NEXT: uzp1 z3.s, z3.s, z3.s ; VBITS_GE_256-NEXT: mov v1.d[1], v0.d[0] +; VBITS_GE_256-NEXT: uzp1 z3.s, z3.s, z3.s ; VBITS_GE_256-NEXT: splice z3.s, p0, z3.s, z2.s ; VBITS_GE_256-NEXT: ptrue p0.h, vl8 ; VBITS_GE_256-NEXT: uzp1 z2.h, z3.h, z3.h @@ -246,11 +246,11 @@ define void @masked_store_trunc_v8i64i32(ptr %ap, ptr %bp, ptr %dest) #0 { ; VBITS_GE_256-NEXT: mov z3.d, p0/z, #-1 // =0xffffffffffffffff ; VBITS_GE_256-NEXT: ptrue p0.s, vl4 ; VBITS_GE_256-NEXT: uzp1 z2.s, z2.s, z2.s -; VBITS_GE_256-NEXT: uzp1 z3.s, z3.s, z3.s ; VBITS_GE_256-NEXT: splice z1.s, p0, z1.s, z0.s +; VBITS_GE_256-NEXT: uzp1 z3.s, z3.s, z3.s ; VBITS_GE_256-NEXT: splice z3.s, p0, z3.s, z2.s -; VBITS_GE_256-NEXT: cmpne p0.s, p1/z, z3.s, #0 -; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x2] +; VBITS_GE_256-NEXT: cmpne p1.s, p1/z, z3.s, #0 +; VBITS_GE_256-NEXT: st1w { z1.s }, p1, [x2] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: masked_store_trunc_v8i64i32: @@ -282,14 +282,14 @@ define void @masked_store_trunc_v16i32i8(ptr %ap, ptr %bp, ptr %dest) #0 { ; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_256-NEXT: cmpeq p0.s, p0/z, z1.s, z3.s ; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h +; VBITS_GE_256-NEXT: uzp1 z0.b, z0.b, z0.b ; VBITS_GE_256-NEXT: mov z2.s, p1/z, #-1 // =0xffffffffffffffff +; VBITS_GE_256-NEXT: uzp1 z1.b, z1.b, z1.b ; VBITS_GE_256-NEXT: mov z3.s, p0/z, #-1 // =0xffffffffffffffff ; VBITS_GE_256-NEXT: ptrue p0.b, vl16 -; VBITS_GE_256-NEXT: uzp1 z0.b, z0.b, z0.b -; VBITS_GE_256-NEXT: uzp1 z1.b, z1.b, z1.b ; VBITS_GE_256-NEXT: uzp1 z2.h, z2.h, z2.h -; VBITS_GE_256-NEXT: uzp1 z3.h, z3.h, z3.h ; VBITS_GE_256-NEXT: mov v1.d[1], v0.d[0] +; VBITS_GE_256-NEXT: uzp1 z3.h, z3.h, z3.h ; VBITS_GE_256-NEXT: uzp1 z2.b, z2.b, z2.b ; VBITS_GE_256-NEXT: uzp1 z3.b, z3.b, z3.b ; VBITS_GE_256-NEXT: mov v3.d[1], v2.d[0] @@ -327,17 +327,17 @@ define void @masked_store_trunc_v16i32i16(ptr %ap, ptr %bp, ptr %dest) #0 { ; VBITS_GE_256-NEXT: cmpeq p0.s, p0/z, z1.s, z3.s ; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h ; VBITS_GE_256-NEXT: mov z2.s, p1/z, #-1 // =0xffffffffffffffff -; VBITS_GE_256-NEXT: ptrue p1.h, vl16 +; VBITS_GE_256-NEXT: ptrue p1.h, vl8 ; VBITS_GE_256-NEXT: mov z3.s, p0/z, #-1 // =0xffffffffffffffff -; VBITS_GE_256-NEXT: ptrue p0.h, vl8 +; VBITS_GE_256-NEXT: ptrue p0.h, vl16 +; VBITS_GE_256-NEXT: splice z1.h, p1, z1.h, z0.h ; VBITS_GE_256-NEXT: uzp1 z2.h, z2.h, z2.h ; VBITS_GE_256-NEXT: uzp1 z3.h, z3.h, z3.h ; VBITS_GE_256-NEXT: uzp1 z2.b, z2.b, z2.b -; VBITS_GE_256-NEXT: splice z1.h, p0, z1.h, z0.h ; VBITS_GE_256-NEXT: uzp1 z3.b, z3.b, z3.b ; VBITS_GE_256-NEXT: mov v3.d[1], v2.d[0] ; VBITS_GE_256-NEXT: sunpklo z2.h, z3.b -; VBITS_GE_256-NEXT: cmpne p0.h, p1/z, z2.h, #0 +; VBITS_GE_256-NEXT: cmpne p0.h, p0/z, z2.h, #0 ; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x2] ; VBITS_GE_256-NEXT: ret ; @@ -375,11 +375,11 @@ define void @masked_store_trunc_v32i16i8(ptr %ap, ptr %bp, ptr %dest) #0 { ; VBITS_GE_256-NEXT: mov z3.h, p0/z, #-1 // =0xffffffffffffffff ; VBITS_GE_256-NEXT: ptrue p0.b, vl16 ; VBITS_GE_256-NEXT: uzp1 z2.b, z2.b, z2.b -; VBITS_GE_256-NEXT: uzp1 z3.b, z3.b, z3.b ; VBITS_GE_256-NEXT: splice z1.b, p0, z1.b, z0.b +; VBITS_GE_256-NEXT: uzp1 z3.b, z3.b, z3.b ; VBITS_GE_256-NEXT: splice z3.b, p0, z3.b, z2.b -; VBITS_GE_256-NEXT: cmpne p0.b, p1/z, z3.b, #0 -; VBITS_GE_256-NEXT: st1b { z1.b }, p0, [x2] +; VBITS_GE_256-NEXT: cmpne p1.b, p1/z, z3.b, #0 +; VBITS_GE_256-NEXT: st1b { z1.b }, p1, [x2] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: masked_store_trunc_v32i16i8: diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll index a5303c901b80..fb169491b0c9 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll @@ -64,7 +64,6 @@ define void @crash_when_lowering_extract_shuffle(ptr %dst, i1 %cond) vscale_rang ; CHECK-NEXT: uunpklo z0.h, z0.b ; CHECK-NEXT: mov x8, #16 // =0x10 ; CHECK-NEXT: mov x10, #8 // =0x8 -; CHECK-NEXT: ld1w { z4.s }, p0/z, [x0, x8, lsl #2] ; CHECK-NEXT: mov v2.b[7], w11 ; CHECK-NEXT: mov v1.b[7], w9 ; CHECK-NEXT: uunpklo z3.h, z3.b @@ -86,22 +85,23 @@ define void @crash_when_lowering_extract_shuffle(ptr %dst, i1 %cond) vscale_rang ; CHECK-NEXT: asr z2.s, z2.s, #31 ; CHECK-NEXT: asr z1.s, z1.s, #31 ; CHECK-NEXT: cmpne p1.s, p0/z, z0.s, #0 -; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, x9, lsl #2] +; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2] ; CHECK-NEXT: cmpne p2.s, p0/z, z3.s, #0 -; CHECK-NEXT: ld1w { z3.s }, p0/z, [x0, x10, lsl #2] +; CHECK-NEXT: ld1w { z3.s }, p0/z, [x0, x9, lsl #2] ; CHECK-NEXT: and z2.s, z2.s, #0x1 ; CHECK-NEXT: and z1.s, z1.s, #0x1 -; CHECK-NEXT: mov z4.s, p1/m, #0 // =0x0 -; CHECK-NEXT: mov z0.s, p2/m, #0 // =0x0 +; CHECK-NEXT: mov z0.s, p1/m, #0 // =0x0 ; CHECK-NEXT: cmpne p3.s, p0/z, z2.s, #0 -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x0] -; CHECK-NEXT: cmpne p1.s, p0/z, z1.s, #0 -; CHECK-NEXT: st1w { z4.s }, p0, [x0, x8, lsl #2] -; CHECK-NEXT: st1w { z0.s }, p0, [x0, x9, lsl #2] -; CHECK-NEXT: mov z3.s, p3/m, #0 // =0x0 -; CHECK-NEXT: mov z2.s, p1/m, #0 // =0x0 -; CHECK-NEXT: st1w { z3.s }, p0, [x0, x10, lsl #2] -; CHECK-NEXT: st1w { z2.s }, p0, [x0] +; CHECK-NEXT: cmpne p4.s, p0/z, z1.s, #0 +; CHECK-NEXT: ld1w { z2.s }, p0/z, [x0, x10, lsl #2] +; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] +; CHECK-NEXT: mov z3.s, p2/m, #0 // =0x0 +; CHECK-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2] +; CHECK-NEXT: mov z2.s, p3/m, #0 // =0x0 +; CHECK-NEXT: mov z1.s, p4/m, #0 // =0x0 +; CHECK-NEXT: st1w { z3.s }, p0, [x0, x9, lsl #2] +; CHECK-NEXT: st1w { z2.s }, p0, [x0, x10, lsl #2] +; CHECK-NEXT: st1w { z1.s }, p0, [x0] ; CHECK-NEXT: .LBB1_2: // %exit ; CHECK-NEXT: ret %broadcast.splat = shufflevector <32 x i1> zeroinitializer, <32 x i1> zeroinitializer, <32 x i32> zeroinitializer diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-splat-vector.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-splat-vector.ll index f97ca05f3bdd..b633057be139 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-splat-vector.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-splat-vector.ll @@ -34,8 +34,8 @@ define <16 x i8> @splat_v16i8(i8 %a) vscale_range(2,0) #0 { define void @splat_v32i8(i8 %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: splat_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl32 ; CHECK-NEXT: mov z0.b, w0 +; CHECK-NEXT: ptrue p0.b, vl32 ; CHECK-NEXT: st1b { z0.b }, p0, [x1] ; CHECK-NEXT: ret %insert = insertelement <32 x i8> undef, i8 %a, i64 0 @@ -47,8 +47,8 @@ define void @splat_v32i8(i8 %a, ptr %b) vscale_range(2,0) #0 { define void @splat_v64i8(i8 %a, ptr %b) #0 { ; VBITS_GE_256-LABEL: splat_v64i8: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.b, vl32 ; VBITS_GE_256-NEXT: mov z0.b, w0 +; VBITS_GE_256-NEXT: ptrue p0.b, vl32 ; VBITS_GE_256-NEXT: mov w8, #32 // =0x20 ; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x1, x8] ; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x1] @@ -56,8 +56,8 @@ define void @splat_v64i8(i8 %a, ptr %b) #0 { ; ; VBITS_GE_512-LABEL: splat_v64i8: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.b, vl64 ; VBITS_GE_512-NEXT: mov z0.b, w0 +; VBITS_GE_512-NEXT: ptrue p0.b, vl64 ; VBITS_GE_512-NEXT: st1b { z0.b }, p0, [x1] ; VBITS_GE_512-NEXT: ret %insert = insertelement <64 x i8> undef, i8 %a, i64 0 @@ -69,8 +69,8 @@ define void @splat_v64i8(i8 %a, ptr %b) #0 { define void @splat_v128i8(i8 %a, ptr %b) vscale_range(8,0) #0 { ; CHECK-LABEL: splat_v128i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl128 ; CHECK-NEXT: mov z0.b, w0 +; CHECK-NEXT: ptrue p0.b, vl128 ; CHECK-NEXT: st1b { z0.b }, p0, [x1] ; CHECK-NEXT: ret %insert = insertelement <128 x i8> undef, i8 %a, i64 0 @@ -82,8 +82,8 @@ define void @splat_v128i8(i8 %a, ptr %b) vscale_range(8,0) #0 { define void @splat_v256i8(i8 %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-LABEL: splat_v256i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl256 ; CHECK-NEXT: mov z0.b, w0 +; CHECK-NEXT: ptrue p0.b, vl256 ; CHECK-NEXT: st1b { z0.b }, p0, [x1] ; CHECK-NEXT: ret %insert = insertelement <256 x i8> undef, i8 %a, i64 0 @@ -117,8 +117,8 @@ define <8 x i16> @splat_v8i16(i16 %a) vscale_range(2,0) #0 { define void @splat_v16i16(i16 %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: splat_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl16 ; CHECK-NEXT: mov z0.h, w0 +; CHECK-NEXT: ptrue p0.h, vl16 ; CHECK-NEXT: st1h { z0.h }, p0, [x1] ; CHECK-NEXT: ret %insert = insertelement <16 x i16> undef, i16 %a, i64 0 @@ -130,8 +130,8 @@ define void @splat_v16i16(i16 %a, ptr %b) vscale_range(2,0) #0 { define void @splat_v32i16(i16 %a, ptr %b) #0 { ; VBITS_GE_256-LABEL: splat_v32i16: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.h, vl16 ; VBITS_GE_256-NEXT: mov z0.h, w0 +; VBITS_GE_256-NEXT: ptrue p0.h, vl16 ; VBITS_GE_256-NEXT: mov x8, #16 // =0x10 ; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x1, x8, lsl #1] ; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x1] @@ -139,8 +139,8 @@ define void @splat_v32i16(i16 %a, ptr %b) #0 { ; ; VBITS_GE_512-LABEL: splat_v32i16: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.h, vl32 ; VBITS_GE_512-NEXT: mov z0.h, w0 +; VBITS_GE_512-NEXT: ptrue p0.h, vl32 ; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x1] ; VBITS_GE_512-NEXT: ret %insert = insertelement <32 x i16> undef, i16 %a, i64 0 @@ -152,8 +152,8 @@ define void @splat_v32i16(i16 %a, ptr %b) #0 { define void @splat_v64i16(i16 %a, ptr %b) vscale_range(8,0) #0 { ; CHECK-LABEL: splat_v64i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl64 ; CHECK-NEXT: mov z0.h, w0 +; CHECK-NEXT: ptrue p0.h, vl64 ; CHECK-NEXT: st1h { z0.h }, p0, [x1] ; CHECK-NEXT: ret %insert = insertelement <64 x i16> undef, i16 %a, i64 0 @@ -165,8 +165,8 @@ define void @splat_v64i16(i16 %a, ptr %b) vscale_range(8,0) #0 { define void @splat_v128i16(i16 %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-LABEL: splat_v128i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl128 ; CHECK-NEXT: mov z0.h, w0 +; CHECK-NEXT: ptrue p0.h, vl128 ; CHECK-NEXT: st1h { z0.h }, p0, [x1] ; CHECK-NEXT: ret %insert = insertelement <128 x i16> undef, i16 %a, i64 0 @@ -200,8 +200,8 @@ define <4 x i32> @splat_v4i32(i32 %a) vscale_range(2,0) #0 { define void @splat_v8i32(i32 %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: splat_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: mov z0.s, w0 +; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: st1w { z0.s }, p0, [x1] ; CHECK-NEXT: ret %insert = insertelement <8 x i32> undef, i32 %a, i64 0 @@ -213,8 +213,8 @@ define void @splat_v8i32(i32 %a, ptr %b) vscale_range(2,0) #0 { define void @splat_v16i32(i32 %a, ptr %b) #0 { ; VBITS_GE_256-LABEL: splat_v16i32: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: mov z0.s, w0 +; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: mov x8, #8 // =0x8 ; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x1, x8, lsl #2] ; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x1] @@ -222,8 +222,8 @@ define void @splat_v16i32(i32 %a, ptr %b) #0 { ; ; VBITS_GE_512-LABEL: splat_v16i32: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.s, vl16 ; VBITS_GE_512-NEXT: mov z0.s, w0 +; VBITS_GE_512-NEXT: ptrue p0.s, vl16 ; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x1] ; VBITS_GE_512-NEXT: ret %insert = insertelement <16 x i32> undef, i32 %a, i64 0 @@ -235,8 +235,8 @@ define void @splat_v16i32(i32 %a, ptr %b) #0 { define void @splat_v32i32(i32 %a, ptr %b) vscale_range(8,0) #0 { ; CHECK-LABEL: splat_v32i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl32 ; CHECK-NEXT: mov z0.s, w0 +; CHECK-NEXT: ptrue p0.s, vl32 ; CHECK-NEXT: st1w { z0.s }, p0, [x1] ; CHECK-NEXT: ret %insert = insertelement <32 x i32> undef, i32 %a, i64 0 @@ -248,8 +248,8 @@ define void @splat_v32i32(i32 %a, ptr %b) vscale_range(8,0) #0 { define void @splat_v64i32(i32 %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-LABEL: splat_v64i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl64 ; CHECK-NEXT: mov z0.s, w0 +; CHECK-NEXT: ptrue p0.s, vl64 ; CHECK-NEXT: st1w { z0.s }, p0, [x1] ; CHECK-NEXT: ret %insert = insertelement <64 x i32> undef, i32 %a, i64 0 @@ -283,8 +283,8 @@ define <2 x i64> @splat_v2i64(i64 %a) vscale_range(2,0) #0 { define void @splat_v4i64(i64 %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: splat_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: mov z0.d, x0 +; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: st1d { z0.d }, p0, [x1] ; CHECK-NEXT: ret %insert = insertelement <4 x i64> undef, i64 %a, i64 0 @@ -296,8 +296,8 @@ define void @splat_v4i64(i64 %a, ptr %b) vscale_range(2,0) #0 { define void @splat_v8i64(i64 %a, ptr %b) #0 { ; VBITS_GE_256-LABEL: splat_v8i64: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: mov z0.d, x0 +; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 ; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x1, x8, lsl #3] ; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x1] @@ -305,8 +305,8 @@ define void @splat_v8i64(i64 %a, ptr %b) #0 { ; ; VBITS_GE_512-LABEL: splat_v8i64: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.d, vl8 ; VBITS_GE_512-NEXT: mov z0.d, x0 +; VBITS_GE_512-NEXT: ptrue p0.d, vl8 ; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x1] ; VBITS_GE_512-NEXT: ret %insert = insertelement <8 x i64> undef, i64 %a, i64 0 @@ -318,8 +318,8 @@ define void @splat_v8i64(i64 %a, ptr %b) #0 { define void @splat_v16i64(i64 %a, ptr %b) vscale_range(8,0) #0 { ; CHECK-LABEL: splat_v16i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl16 ; CHECK-NEXT: mov z0.d, x0 +; CHECK-NEXT: ptrue p0.d, vl16 ; CHECK-NEXT: st1d { z0.d }, p0, [x1] ; CHECK-NEXT: ret %insert = insertelement <16 x i64> undef, i64 %a, i64 0 @@ -331,8 +331,8 @@ define void @splat_v16i64(i64 %a, ptr %b) vscale_range(8,0) #0 { define void @splat_v32i64(i64 %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-LABEL: splat_v32i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl32 ; CHECK-NEXT: mov z0.d, x0 +; CHECK-NEXT: ptrue p0.d, vl32 ; CHECK-NEXT: st1d { z0.d }, p0, [x1] ; CHECK-NEXT: ret %insert = insertelement <32 x i64> undef, i64 %a, i64 0 @@ -372,8 +372,8 @@ define <8 x half> @splat_v8f16(half %a) vscale_range(2,0) #0 { define void @splat_v16f16(half %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: splat_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl16 ; CHECK-NEXT: // kill: def $h0 killed $h0 def $z0 +; CHECK-NEXT: ptrue p0.h, vl16 ; CHECK-NEXT: mov z0.h, h0 ; CHECK-NEXT: st1h { z0.h }, p0, [x0] ; CHECK-NEXT: ret @@ -386,8 +386,8 @@ define void @splat_v16f16(half %a, ptr %b) vscale_range(2,0) #0 { define void @splat_v32f16(half %a, ptr %b) #0 { ; VBITS_GE_256-LABEL: splat_v32f16: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.h, vl16 ; VBITS_GE_256-NEXT: // kill: def $h0 killed $h0 def $z0 +; VBITS_GE_256-NEXT: ptrue p0.h, vl16 ; VBITS_GE_256-NEXT: mov x8, #16 // =0x10 ; VBITS_GE_256-NEXT: mov z0.h, h0 ; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0, x8, lsl #1] @@ -396,8 +396,8 @@ define void @splat_v32f16(half %a, ptr %b) #0 { ; ; VBITS_GE_512-LABEL: splat_v32f16: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.h, vl32 ; VBITS_GE_512-NEXT: // kill: def $h0 killed $h0 def $z0 +; VBITS_GE_512-NEXT: ptrue p0.h, vl32 ; VBITS_GE_512-NEXT: mov z0.h, h0 ; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0] ; VBITS_GE_512-NEXT: ret @@ -410,8 +410,8 @@ define void @splat_v32f16(half %a, ptr %b) #0 { define void @splat_v64f16(half %a, ptr %b) vscale_range(8,0) #0 { ; CHECK-LABEL: splat_v64f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl64 ; CHECK-NEXT: // kill: def $h0 killed $h0 def $z0 +; CHECK-NEXT: ptrue p0.h, vl64 ; CHECK-NEXT: mov z0.h, h0 ; CHECK-NEXT: st1h { z0.h }, p0, [x0] ; CHECK-NEXT: ret @@ -424,8 +424,8 @@ define void @splat_v64f16(half %a, ptr %b) vscale_range(8,0) #0 { define void @splat_v128f16(half %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-LABEL: splat_v128f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl128 ; CHECK-NEXT: // kill: def $h0 killed $h0 def $z0 +; CHECK-NEXT: ptrue p0.h, vl128 ; CHECK-NEXT: mov z0.h, h0 ; CHECK-NEXT: st1h { z0.h }, p0, [x0] ; CHECK-NEXT: ret @@ -462,8 +462,8 @@ define <4 x float> @splat_v4f32(float %a, <4 x float> %op2) vscale_range(2,0) #0 define void @splat_v8f32(float %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: splat_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: // kill: def $s0 killed $s0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: mov z0.s, s0 ; CHECK-NEXT: st1w { z0.s }, p0, [x0] ; CHECK-NEXT: ret @@ -476,8 +476,8 @@ define void @splat_v8f32(float %a, ptr %b) vscale_range(2,0) #0 { define void @splat_v16f32(float %a, ptr %b) #0 { ; VBITS_GE_256-LABEL: splat_v16f32: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: // kill: def $s0 killed $s0 def $z0 +; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: mov x8, #8 // =0x8 ; VBITS_GE_256-NEXT: mov z0.s, s0 ; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2] @@ -486,8 +486,8 @@ define void @splat_v16f32(float %a, ptr %b) #0 { ; ; VBITS_GE_512-LABEL: splat_v16f32: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.s, vl16 ; VBITS_GE_512-NEXT: // kill: def $s0 killed $s0 def $z0 +; VBITS_GE_512-NEXT: ptrue p0.s, vl16 ; VBITS_GE_512-NEXT: mov z0.s, s0 ; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0] ; VBITS_GE_512-NEXT: ret @@ -500,8 +500,8 @@ define void @splat_v16f32(float %a, ptr %b) #0 { define void @splat_v32f32(float %a, ptr %b) vscale_range(8,0) #0 { ; CHECK-LABEL: splat_v32f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl32 ; CHECK-NEXT: // kill: def $s0 killed $s0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl32 ; CHECK-NEXT: mov z0.s, s0 ; CHECK-NEXT: st1w { z0.s }, p0, [x0] ; CHECK-NEXT: ret @@ -514,8 +514,8 @@ define void @splat_v32f32(float %a, ptr %b) vscale_range(8,0) #0 { define void @splat_v64f32(float %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-LABEL: splat_v64f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl64 ; CHECK-NEXT: // kill: def $s0 killed $s0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl64 ; CHECK-NEXT: mov z0.s, s0 ; CHECK-NEXT: st1w { z0.s }, p0, [x0] ; CHECK-NEXT: ret @@ -550,8 +550,8 @@ define <2 x double> @splat_v2f64(double %a, <2 x double> %op2) vscale_range(2,0) define void @splat_v4f64(double %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: splat_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: mov z0.d, d0 ; CHECK-NEXT: st1d { z0.d }, p0, [x0] ; CHECK-NEXT: ret @@ -564,8 +564,8 @@ define void @splat_v4f64(double %a, ptr %b) vscale_range(2,0) #0 { define void @splat_v8f64(double %a, ptr %b) #0 { ; VBITS_GE_256-LABEL: splat_v8f64: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: // kill: def $d0 killed $d0 def $z0 +; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 ; VBITS_GE_256-NEXT: mov z0.d, d0 ; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3] @@ -574,8 +574,8 @@ define void @splat_v8f64(double %a, ptr %b) #0 { ; ; VBITS_GE_512-LABEL: splat_v8f64: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.d, vl8 ; VBITS_GE_512-NEXT: // kill: def $d0 killed $d0 def $z0 +; VBITS_GE_512-NEXT: ptrue p0.d, vl8 ; VBITS_GE_512-NEXT: mov z0.d, d0 ; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0] ; VBITS_GE_512-NEXT: ret @@ -588,8 +588,8 @@ define void @splat_v8f64(double %a, ptr %b) #0 { define void @splat_v16f64(double %a, ptr %b) vscale_range(8,0) #0 { ; CHECK-LABEL: splat_v16f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl16 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.d, vl16 ; CHECK-NEXT: mov z0.d, d0 ; CHECK-NEXT: st1d { z0.d }, p0, [x0] ; CHECK-NEXT: ret @@ -602,8 +602,8 @@ define void @splat_v16f64(double %a, ptr %b) vscale_range(8,0) #0 { define void @splat_v32f64(double %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-LABEL: splat_v32f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl32 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.d, vl32 ; CHECK-NEXT: mov z0.d, d0 ; CHECK-NEXT: st1d { z0.d }, p0, [x0] ; CHECK-NEXT: ret @@ -620,8 +620,8 @@ define void @splat_v32f64(double %a, ptr %b) vscale_range(16,0) #0 { define void @splat_imm_v64i8(ptr %a) vscale_range(4,0) #0 { ; CHECK-LABEL: splat_imm_v64i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl64 ; CHECK-NEXT: mov z0.b, #1 // =0x1 +; CHECK-NEXT: ptrue p0.b, vl64 ; CHECK-NEXT: st1b { z0.b }, p0, [x0] ; CHECK-NEXT: ret %insert = insertelement <64 x i8> undef, i8 1, i64 0 @@ -633,8 +633,8 @@ define void @splat_imm_v64i8(ptr %a) vscale_range(4,0) #0 { define void @splat_imm_v32i16(ptr %a) vscale_range(4,0) #0 { ; CHECK-LABEL: splat_imm_v32i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl32 ; CHECK-NEXT: mov z0.h, #2 // =0x2 +; CHECK-NEXT: ptrue p0.h, vl32 ; CHECK-NEXT: st1h { z0.h }, p0, [x0] ; CHECK-NEXT: ret %insert = insertelement <32 x i16> undef, i16 2, i64 0 @@ -646,8 +646,8 @@ define void @splat_imm_v32i16(ptr %a) vscale_range(4,0) #0 { define void @splat_imm_v16i32(ptr %a) vscale_range(4,0) #0 { ; CHECK-LABEL: splat_imm_v16i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl16 ; CHECK-NEXT: mov z0.s, #3 // =0x3 +; CHECK-NEXT: ptrue p0.s, vl16 ; CHECK-NEXT: st1w { z0.s }, p0, [x0] ; CHECK-NEXT: ret %insert = insertelement <16 x i32> undef, i32 3, i64 0 @@ -659,8 +659,8 @@ define void @splat_imm_v16i32(ptr %a) vscale_range(4,0) #0 { define void @splat_imm_v8i64(ptr %a) vscale_range(4,0) #0 { ; CHECK-LABEL: splat_imm_v8i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl8 ; CHECK-NEXT: mov z0.d, #4 // =0x4 +; CHECK-NEXT: ptrue p0.d, vl8 ; CHECK-NEXT: st1d { z0.d }, p0, [x0] ; CHECK-NEXT: ret %insert = insertelement <8 x i64> undef, i64 4, i64 0 @@ -676,8 +676,8 @@ define void @splat_imm_v8i64(ptr %a) vscale_range(4,0) #0 { define void @splat_imm_v32f16(ptr %a) vscale_range(4,0) #0 { ; CHECK-LABEL: splat_imm_v32f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl32 ; CHECK-NEXT: fmov z0.h, #5.00000000 +; CHECK-NEXT: ptrue p0.h, vl32 ; CHECK-NEXT: st1h { z0.h }, p0, [x0] ; CHECK-NEXT: ret %insert = insertelement <32 x half> undef, half 5.0, i64 0 @@ -689,8 +689,8 @@ define void @splat_imm_v32f16(ptr %a) vscale_range(4,0) #0 { define void @splat_imm_v16f32(ptr %a) vscale_range(4,0) #0 { ; CHECK-LABEL: splat_imm_v16f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl16 ; CHECK-NEXT: fmov z0.s, #6.00000000 +; CHECK-NEXT: ptrue p0.s, vl16 ; CHECK-NEXT: st1w { z0.s }, p0, [x0] ; CHECK-NEXT: ret %insert = insertelement <16 x float> undef, float 6.0, i64 0 @@ -702,8 +702,8 @@ define void @splat_imm_v16f32(ptr %a) vscale_range(4,0) #0 { define void @splat_imm_v8f64(ptr %a) vscale_range(4,0) #0 { ; CHECK-LABEL: splat_imm_v8f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl8 ; CHECK-NEXT: fmov z0.d, #7.00000000 +; CHECK-NEXT: ptrue p0.d, vl8 ; CHECK-NEXT: st1d { z0.d }, p0, [x0] ; CHECK-NEXT: ret %insert = insertelement <8 x double> undef, double 7.0, i64 0 diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-trunc-stores.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-trunc-stores.ll index 2dc4bddb81a6..020d5cb53bf2 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-trunc-stores.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-trunc-stores.ll @@ -36,14 +36,14 @@ define void @store_trunc_v8i64i8(ptr %ap, ptr %dest) #0 { ; VBITS_GE_256: // %bb.0: ; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 -; VBITS_GE_256-NEXT: ptrue p1.s, vl8 ; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] ; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0] ; VBITS_GE_256-NEXT: ptrue p0.s, vl4 ; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s ; VBITS_GE_256-NEXT: splice z1.s, p0, z1.s, z0.s -; VBITS_GE_256-NEXT: st1b { z1.s }, p1, [x1] +; VBITS_GE_256-NEXT: ptrue p0.s, vl8 +; VBITS_GE_256-NEXT: st1b { z1.s }, p0, [x1] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: store_trunc_v8i64i8: @@ -117,14 +117,14 @@ define void @store_trunc_v8i64i32(ptr %ap, ptr %dest) #0 { ; VBITS_GE_256: // %bb.0: ; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 -; VBITS_GE_256-NEXT: ptrue p1.s, vl8 ; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] ; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0] ; VBITS_GE_256-NEXT: ptrue p0.s, vl4 ; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s ; VBITS_GE_256-NEXT: splice z1.s, p0, z1.s, z0.s -; VBITS_GE_256-NEXT: st1w { z1.s }, p1, [x1] +; VBITS_GE_256-NEXT: ptrue p0.s, vl8 +; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x1] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: store_trunc_v8i64i32: @@ -172,14 +172,14 @@ define void @store_trunc_v16i32i16(ptr %ap, ptr %dest) #0 { ; VBITS_GE_256: // %bb.0: ; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: mov x8, #8 // =0x8 -; VBITS_GE_256-NEXT: ptrue p1.h, vl16 ; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2] ; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0] ; VBITS_GE_256-NEXT: ptrue p0.h, vl8 ; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h ; VBITS_GE_256-NEXT: splice z1.h, p0, z1.h, z0.h -; VBITS_GE_256-NEXT: st1h { z1.h }, p1, [x1] +; VBITS_GE_256-NEXT: ptrue p0.h, vl16 +; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x1] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: store_trunc_v16i32i16: @@ -199,14 +199,14 @@ define void @store_trunc_v32i16i8(ptr %ap, ptr %dest) #0 { ; VBITS_GE_256: // %bb.0: ; VBITS_GE_256-NEXT: ptrue p0.h, vl16 ; VBITS_GE_256-NEXT: mov x8, #16 // =0x10 -; VBITS_GE_256-NEXT: ptrue p1.b, vl32 ; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1] ; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x0] ; VBITS_GE_256-NEXT: ptrue p0.b, vl16 ; VBITS_GE_256-NEXT: uzp1 z0.b, z0.b, z0.b ; VBITS_GE_256-NEXT: uzp1 z1.b, z1.b, z1.b ; VBITS_GE_256-NEXT: splice z1.b, p0, z1.b, z0.b -; VBITS_GE_256-NEXT: st1b { z1.b }, p1, [x1] +; VBITS_GE_256-NEXT: ptrue p0.b, vl32 +; VBITS_GE_256-NEXT: st1b { z1.b }, p0, [x1] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: store_trunc_v32i16i8: diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll index 68c234a20d11..28094c7b68e7 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll @@ -559,13 +559,13 @@ define <8 x i16> @shuffle_index_indices_from_both_ops_i16(ptr %a, ptr %b) { ; ; SVE2_128_NOMAX-LABEL: shuffle_index_indices_from_both_ops_i16: ; SVE2_128_NOMAX: // %bb.0: -; SVE2_128_NOMAX-NEXT: ptrue p0.h, vl8 ; SVE2_128_NOMAX-NEXT: cnth x8 ; SVE2_128_NOMAX-NEXT: adrp x9, .LCPI7_0 ; SVE2_128_NOMAX-NEXT: adrp x10, .LCPI7_1 ; SVE2_128_NOMAX-NEXT: mov z0.h, w8 ; SVE2_128_NOMAX-NEXT: ldr q1, [x9, :lo12:.LCPI7_0] ; SVE2_128_NOMAX-NEXT: ldr q2, [x10, :lo12:.LCPI7_1] +; SVE2_128_NOMAX-NEXT: ptrue p0.h, vl8 ; SVE2_128_NOMAX-NEXT: mad z0.h, p0/m, z1.h, z2.h ; SVE2_128_NOMAX-NEXT: ldr q1, [x0] ; SVE2_128_NOMAX-NEXT: ldr q2, [x1] @@ -575,13 +575,13 @@ define <8 x i16> @shuffle_index_indices_from_both_ops_i16(ptr %a, ptr %b) { ; ; SVE2_NOMIN_NOMAX-LABEL: shuffle_index_indices_from_both_ops_i16: ; SVE2_NOMIN_NOMAX: // %bb.0: -; SVE2_NOMIN_NOMAX-NEXT: ptrue p0.h, vl8 ; SVE2_NOMIN_NOMAX-NEXT: cnth x8 ; SVE2_NOMIN_NOMAX-NEXT: adrp x9, .LCPI7_0 ; SVE2_NOMIN_NOMAX-NEXT: adrp x10, .LCPI7_1 ; SVE2_NOMIN_NOMAX-NEXT: mov z0.h, w8 ; SVE2_NOMIN_NOMAX-NEXT: ldr q1, [x9, :lo12:.LCPI7_0] ; SVE2_NOMIN_NOMAX-NEXT: ldr q2, [x10, :lo12:.LCPI7_1] +; SVE2_NOMIN_NOMAX-NEXT: ptrue p0.h, vl8 ; SVE2_NOMIN_NOMAX-NEXT: mad z0.h, p0/m, z1.h, z2.h ; SVE2_NOMIN_NOMAX-NEXT: ldr q1, [x0] ; SVE2_NOMIN_NOMAX-NEXT: ldr q2, [x1] @@ -597,9 +597,9 @@ define <8 x i16> @shuffle_index_indices_from_both_ops_i16(ptr %a, ptr %b) { ; SVE2_MIN_256_NOMAX-NEXT: adrp x9, .LCPI7_1 ; SVE2_MIN_256_NOMAX-NEXT: add x9, x9, :lo12:.LCPI7_1 ; SVE2_MIN_256_NOMAX-NEXT: cnth x10 -; SVE2_MIN_256_NOMAX-NEXT: mov z2.h, w10 ; SVE2_MIN_256_NOMAX-NEXT: ld1h { z0.h }, p0/z, [x8] ; SVE2_MIN_256_NOMAX-NEXT: ld1h { z1.h }, p0/z, [x9] +; SVE2_MIN_256_NOMAX-NEXT: mov z2.h, w10 ; SVE2_MIN_256_NOMAX-NEXT: mad z0.h, p0/m, z2.h, z1.h ; SVE2_MIN_256_NOMAX-NEXT: ldr q1, [x0] ; SVE2_MIN_256_NOMAX-NEXT: ldr q2, [x1] diff --git a/llvm/test/CodeGen/AArch64/sve-fp-int-min-max.ll b/llvm/test/CodeGen/AArch64/sve-fp-int-min-max.ll index 5ff9f0f0df62..afe13851f0b9 100644 --- a/llvm/test/CodeGen/AArch64/sve-fp-int-min-max.ll +++ b/llvm/test/CodeGen/AArch64/sve-fp-int-min-max.ll @@ -7,11 +7,11 @@ define i64 @scalable_int_min_max(ptr %arg, ptr %arg1, %i37, < ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, #3745 // =0xea1 ; CHECK-NEXT: movk w8, #16618, lsl #16 +; CHECK-NEXT: ld1w { z3.d }, p0/z, [x0] ; CHECK-NEXT: mov z4.s, w8 ; CHECK-NEXT: mov w8, #57344 // =0xe000 ; CHECK-NEXT: movk w8, #17535, lsl #16 ; CHECK-NEXT: mov z5.s, w8 -; CHECK-NEXT: ld1w { z3.d }, p0/z, [x0] ; CHECK-NEXT: fmul z4.s, p0/m, z4.s, z3.s ; CHECK-NEXT: fadd z4.s, p0/m, z4.s, z5.s ; CHECK-NEXT: mov z5.d, #1023 // =0x3ff diff --git a/llvm/test/CodeGen/AArch64/sve-fp-reciprocal.ll b/llvm/test/CodeGen/AArch64/sve-fp-reciprocal.ll index aefc8de43143..6420071b3dce 100644 --- a/llvm/test/CodeGen/AArch64/sve-fp-reciprocal.ll +++ b/llvm/test/CodeGen/AArch64/sve-fp-reciprocal.ll @@ -92,8 +92,8 @@ define @fsqrt_recip_8f16( %a) #0 { ; CHECK: // %bb.0: ; CHECK-NEXT: frsqrte z1.h, z0.h ; CHECK-NEXT: ptrue p0.h -; CHECK-NEXT: fmul z2.h, z1.h, z1.h ; CHECK-NEXT: fcmne p0.h, p0/z, z0.h, #0.0 +; CHECK-NEXT: fmul z2.h, z1.h, z1.h ; CHECK-NEXT: frsqrts z2.h, z0.h, z2.h ; CHECK-NEXT: fmul z1.h, z1.h, z2.h ; CHECK-NEXT: fmul z0.h, p0/m, z0.h, z1.h @@ -117,8 +117,8 @@ define @fsqrt_recip_4f32( %a) #0 { ; CHECK: // %bb.0: ; CHECK-NEXT: frsqrte z1.s, z0.s ; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: fmul z2.s, z1.s, z1.s ; CHECK-NEXT: fcmne p0.s, p0/z, z0.s, #0.0 +; CHECK-NEXT: fmul z2.s, z1.s, z1.s ; CHECK-NEXT: frsqrts z2.s, z0.s, z2.s ; CHECK-NEXT: fmul z1.s, z1.s, z2.s ; CHECK-NEXT: fmul z2.s, z1.s, z1.s @@ -145,8 +145,8 @@ define @fsqrt_recip_2f64( %a) #0 { ; CHECK: // %bb.0: ; CHECK-NEXT: frsqrte z1.d, z0.d ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: fmul z2.d, z1.d, z1.d ; CHECK-NEXT: fcmne p0.d, p0/z, z0.d, #0.0 +; CHECK-NEXT: fmul z2.d, z1.d, z1.d ; CHECK-NEXT: frsqrts z2.d, z0.d, z2.d ; CHECK-NEXT: fmul z1.d, z1.d, z2.d ; CHECK-NEXT: fmul z2.d, z1.d, z1.d diff --git a/llvm/test/CodeGen/AArch64/sve-fp-reduce-fadda.ll b/llvm/test/CodeGen/AArch64/sve-fp-reduce-fadda.ll index 460d8a8694bc..1a2ab8d4253a 100644 --- a/llvm/test/CodeGen/AArch64/sve-fp-reduce-fadda.ll +++ b/llvm/test/CodeGen/AArch64/sve-fp-reduce-fadda.ll @@ -51,10 +51,10 @@ define half @fadda_nxv6f16( %v, half %s) { ; CHECK-NEXT: addvl sp, sp, #-1 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w8, #32768 // =0x8000 -; CHECK-NEXT: ptrue p1.d +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w8 +; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: st1h { z0.h }, p0, [sp] ; CHECK-NEXT: fmov s0, s1 ; CHECK-NEXT: st1h { z2.d }, p1, [sp, #3, mul vl] diff --git a/llvm/test/CodeGen/AArch64/sve-fptosi-sat.ll b/llvm/test/CodeGen/AArch64/sve-fptosi-sat.ll index 813f1601e809..584c29ebcfc0 100644 --- a/llvm/test/CodeGen/AArch64/sve-fptosi-sat.ll +++ b/llvm/test/CodeGen/AArch64/sve-fptosi-sat.ll @@ -14,8 +14,8 @@ declare @llvm.fptosi.sat.nxv4f32.nxv4i64( define @test_signed_v2f32_v2i32( %f) { ; CHECK-LABEL: test_signed_v2f32_v2i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, #-822083584 // =0xcf000000 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z2.d, #0xffffffff80000000 ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: mov w8, #1325400063 // =0x4effffff @@ -38,8 +38,8 @@ define @test_signed_v2f32_v2i32( %f) { define @test_signed_v4f32_v4i32( %f) { ; CHECK-LABEL: test_signed_v4f32_v4i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #-822083584 // =0xcf000000 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, #0x80000000 ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: mov w8, #1325400063 // =0x4effffff @@ -67,29 +67,29 @@ define @test_signed_v8f32_v8i32( %f) { ; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #-822083584 // =0xcf000000 -; CHECK-NEXT: mov z3.s, #0x80000000 +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: mov z6.s, #0x7fffffff ; CHECK-NEXT: mov z2.s, w8 ; CHECK-NEXT: mov w8, #1325400063 // =0x4effffff -; CHECK-NEXT: mov z6.s, #0x7fffffff -; CHECK-NEXT: mov z4.s, w8 -; CHECK-NEXT: fcmge p1.s, p0/z, z0.s, z2.s -; CHECK-NEXT: fcmge p2.s, p0/z, z1.s, z2.s -; CHECK-NEXT: movprfx z2, z0 -; CHECK-NEXT: fcvtzs z2.s, p0/m, z0.s +; CHECK-NEXT: mov z3.s, w8 +; CHECK-NEXT: movprfx z4, z0 +; CHECK-NEXT: fcvtzs z4.s, p0/m, z0.s ; CHECK-NEXT: movprfx z5, z1 ; CHECK-NEXT: fcvtzs z5.s, p0/m, z1.s -; CHECK-NEXT: fcmgt p3.s, p0/z, z0.s, z4.s -; CHECK-NEXT: fcmgt p4.s, p0/z, z1.s, z4.s +; CHECK-NEXT: fcmge p1.s, p0/z, z0.s, z2.s +; CHECK-NEXT: fcmge p2.s, p0/z, z1.s, z2.s +; CHECK-NEXT: mov z2.s, #0x80000000 +; CHECK-NEXT: fcmgt p3.s, p0/z, z0.s, z3.s +; CHECK-NEXT: fcmgt p4.s, p0/z, z1.s, z3.s ; CHECK-NEXT: not p1.b, p0/z, p1.b ; CHECK-NEXT: not p2.b, p0/z, p2.b -; CHECK-NEXT: mov z2.s, p1/m, z3.s +; CHECK-NEXT: sel z3.s, p1, z2.s, z4.s ; CHECK-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s ; CHECK-NEXT: fcmuo p0.s, p0/z, z1.s, z1.s -; CHECK-NEXT: sel z3.s, p2, z3.s, z5.s -; CHECK-NEXT: sel z0.s, p3, z6.s, z2.s -; CHECK-NEXT: sel z1.s, p4, z6.s, z3.s +; CHECK-NEXT: sel z2.s, p2, z2.s, z5.s +; CHECK-NEXT: sel z0.s, p3, z6.s, z3.s +; CHECK-NEXT: sel z1.s, p4, z6.s, z2.s ; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: mov z0.s, p1/m, #0 // =0x0 ; CHECK-NEXT: mov z1.s, p0/m, #0 // =0x0 @@ -103,8 +103,8 @@ define @test_signed_v8f32_v8i32( %f) { define @test_signed_v4f32_v4i16( %f) { ; CHECK-LABEL: test_signed_v4f32_v4i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #-956301312 // =0xc7000000 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: mov w8, #65024 // =0xfe00 ; CHECK-NEXT: movk w8, #18175, lsl #16 @@ -132,28 +132,28 @@ define @test_signed_v8f32_v8i16( %f) { ; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #-956301312 // =0xc7000000 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z5.s, #32767 // =0x7fff ; CHECK-NEXT: mov z2.s, w8 ; CHECK-NEXT: mov w8, #65024 // =0xfe00 ; CHECK-NEXT: movk w8, #18175, lsl #16 -; CHECK-NEXT: mov z3.s, w8 -; CHECK-NEXT: fcmge p1.s, p0/z, z1.s, z2.s -; CHECK-NEXT: fcmge p2.s, p0/z, z0.s, z2.s -; CHECK-NEXT: movprfx z2, z1 -; CHECK-NEXT: fcvtzs z2.s, p0/m, z1.s +; CHECK-NEXT: movprfx z3, z1 +; CHECK-NEXT: fcvtzs z3.s, p0/m, z1.s ; CHECK-NEXT: movprfx z4, z0 ; CHECK-NEXT: fcvtzs z4.s, p0/m, z0.s -; CHECK-NEXT: fcmgt p3.s, p0/z, z1.s, z3.s -; CHECK-NEXT: fcmgt p4.s, p0/z, z0.s, z3.s +; CHECK-NEXT: fcmge p1.s, p0/z, z1.s, z2.s +; CHECK-NEXT: fcmge p2.s, p0/z, z0.s, z2.s +; CHECK-NEXT: mov z2.s, w8 +; CHECK-NEXT: fcmgt p3.s, p0/z, z1.s, z2.s +; CHECK-NEXT: fcmgt p4.s, p0/z, z0.s, z2.s ; CHECK-NEXT: not p1.b, p0/z, p1.b ; CHECK-NEXT: not p2.b, p0/z, p2.b -; CHECK-NEXT: mov z2.s, p1/m, #-32768 // =0xffffffffffff8000 +; CHECK-NEXT: mov z3.s, p1/m, #-32768 // =0xffffffffffff8000 ; CHECK-NEXT: fcmuo p1.s, p0/z, z1.s, z1.s ; CHECK-NEXT: fcmuo p0.s, p0/z, z0.s, z0.s ; CHECK-NEXT: mov z4.s, p2/m, #-32768 // =0xffffffffffff8000 -; CHECK-NEXT: sel z0.s, p3, z5.s, z2.s +; CHECK-NEXT: sel z0.s, p3, z5.s, z3.s ; CHECK-NEXT: sel z1.s, p4, z5.s, z4.s ; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: mov z0.s, p1/m, #0 // =0x0 @@ -169,8 +169,8 @@ define @test_signed_v8f32_v8i16( %f) { define @test_signed_v2f32_v2i64( %f) { ; CHECK-LABEL: test_signed_v2f32_v2i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, #-553648128 // =0xdf000000 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z2.d, #0x8000000000000000 ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: mov w8, #1593835519 // =0x5effffff @@ -198,31 +198,31 @@ define @test_signed_v4f32_v4i64( %f) { ; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, #-553648128 // =0xdf000000 ; CHECK-NEXT: uunpklo z1.d, z0.s ; CHECK-NEXT: uunpkhi z0.d, z0.s ; CHECK-NEXT: mov z2.s, w8 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, #1593835519 // =0x5effffff -; CHECK-NEXT: mov z3.d, #0x8000000000000000 -; CHECK-NEXT: mov z4.s, w8 +; CHECK-NEXT: mov z3.s, w8 ; CHECK-NEXT: mov z6.d, #0x7fffffffffffffff ; CHECK-NEXT: fcmge p1.s, p0/z, z1.s, z2.s ; CHECK-NEXT: fcmge p2.s, p0/z, z0.s, z2.s -; CHECK-NEXT: movprfx z2, z1 -; CHECK-NEXT: fcvtzs z2.d, p0/m, z1.s +; CHECK-NEXT: mov z2.d, #0x8000000000000000 +; CHECK-NEXT: movprfx z4, z1 +; CHECK-NEXT: fcvtzs z4.d, p0/m, z1.s ; CHECK-NEXT: movprfx z5, z0 ; CHECK-NEXT: fcvtzs z5.d, p0/m, z0.s -; CHECK-NEXT: fcmgt p3.s, p0/z, z1.s, z4.s -; CHECK-NEXT: fcmgt p4.s, p0/z, z0.s, z4.s +; CHECK-NEXT: fcmgt p3.s, p0/z, z1.s, z3.s +; CHECK-NEXT: fcmgt p4.s, p0/z, z0.s, z3.s ; CHECK-NEXT: not p1.b, p0/z, p1.b ; CHECK-NEXT: not p2.b, p0/z, p2.b -; CHECK-NEXT: mov z2.d, p1/m, z3.d +; CHECK-NEXT: sel z3.d, p1, z2.d, z4.d ; CHECK-NEXT: fcmuo p1.s, p0/z, z1.s, z1.s ; CHECK-NEXT: fcmuo p0.s, p0/z, z0.s, z0.s -; CHECK-NEXT: sel z3.d, p2, z3.d, z5.d -; CHECK-NEXT: sel z0.d, p3, z6.d, z2.d -; CHECK-NEXT: sel z1.d, p4, z6.d, z3.d +; CHECK-NEXT: sel z2.d, p2, z2.d, z5.d +; CHECK-NEXT: sel z0.d, p3, z6.d, z3.d +; CHECK-NEXT: sel z1.d, p4, z6.d, z2.d ; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: mov z0.d, p1/m, #0 // =0x0 ; CHECK-NEXT: mov z1.d, p0/m, #0 // =0x0 @@ -246,8 +246,8 @@ declare @llvm.fptosi.sat.nxv4f64.nxv4i64( @test_signed_v2f64_v2i32( %f) { ; CHECK-LABEL: test_signed_v2f64_v2i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, #-4476578029606273024 // =0xc1e0000000000000 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z2.d, #0xffffffff80000000 ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: mov x8, #281474972516352 // =0xffffffc00000 @@ -276,30 +276,30 @@ define @test_signed_v4f64_v4i32( %f) { ; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, #-4476578029606273024 // =0xc1e0000000000000 -; CHECK-NEXT: mov z3.d, #0xffffffff80000000 +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: mov z6.d, #0x7fffffff ; CHECK-NEXT: mov z2.d, x8 ; CHECK-NEXT: mov x8, #281474972516352 // =0xffffffc00000 -; CHECK-NEXT: mov z6.d, #0x7fffffff ; CHECK-NEXT: movk x8, #16863, lsl #48 -; CHECK-NEXT: mov z4.d, x8 -; CHECK-NEXT: fcmge p1.d, p0/z, z1.d, z2.d -; CHECK-NEXT: fcmge p2.d, p0/z, z0.d, z2.d -; CHECK-NEXT: movprfx z2, z1 -; CHECK-NEXT: fcvtzs z2.d, p0/m, z1.d +; CHECK-NEXT: movprfx z4, z1 +; CHECK-NEXT: fcvtzs z4.d, p0/m, z1.d ; CHECK-NEXT: movprfx z5, z0 ; CHECK-NEXT: fcvtzs z5.d, p0/m, z0.d -; CHECK-NEXT: fcmgt p3.d, p0/z, z1.d, z4.d -; CHECK-NEXT: fcmgt p4.d, p0/z, z0.d, z4.d +; CHECK-NEXT: mov z3.d, x8 +; CHECK-NEXT: fcmge p1.d, p0/z, z1.d, z2.d +; CHECK-NEXT: fcmge p2.d, p0/z, z0.d, z2.d +; CHECK-NEXT: mov z2.d, #0xffffffff80000000 +; CHECK-NEXT: fcmgt p3.d, p0/z, z1.d, z3.d +; CHECK-NEXT: fcmgt p4.d, p0/z, z0.d, z3.d ; CHECK-NEXT: not p1.b, p0/z, p1.b ; CHECK-NEXT: not p2.b, p0/z, p2.b -; CHECK-NEXT: mov z2.d, p1/m, z3.d +; CHECK-NEXT: sel z3.d, p1, z2.d, z4.d ; CHECK-NEXT: fcmuo p1.d, p0/z, z1.d, z1.d ; CHECK-NEXT: fcmuo p0.d, p0/z, z0.d, z0.d -; CHECK-NEXT: sel z3.d, p2, z3.d, z5.d -; CHECK-NEXT: sel z0.d, p3, z6.d, z2.d -; CHECK-NEXT: sel z1.d, p4, z6.d, z3.d +; CHECK-NEXT: sel z2.d, p2, z2.d, z5.d +; CHECK-NEXT: sel z0.d, p3, z6.d, z3.d +; CHECK-NEXT: sel z1.d, p4, z6.d, z2.d ; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: mov z0.d, p1/m, #0 // =0x0 ; CHECK-NEXT: mov z1.d, p0/m, #0 // =0x0 @@ -322,48 +322,48 @@ define @test_signed_v8f64_v8i32( %f) { ; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, #-4476578029606273024 // =0xc1e0000000000000 -; CHECK-NEXT: mov z26.d, #0x7fffffff +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: mov z5.d, #0xffffffff80000000 ; CHECK-NEXT: mov z4.d, x8 ; CHECK-NEXT: mov x8, #281474972516352 // =0xffffffc00000 +; CHECK-NEXT: mov z26.d, #0x7fffffff ; CHECK-NEXT: movk x8, #16863, lsl #48 -; CHECK-NEXT: mov z5.d, x8 -; CHECK-NEXT: fcmge p1.d, p0/z, z1.d, z4.d -; CHECK-NEXT: fcmge p2.d, p0/z, z0.d, z4.d -; CHECK-NEXT: fcmge p3.d, p0/z, z3.d, z4.d -; CHECK-NEXT: fcmge p4.d, p0/z, z2.d, z4.d -; CHECK-NEXT: mov z4.d, #0xffffffff80000000 -; CHECK-NEXT: movprfx z6, z1 -; CHECK-NEXT: fcvtzs z6.d, p0/m, z1.d ; CHECK-NEXT: movprfx z7, z0 ; CHECK-NEXT: fcvtzs z7.d, p0/m, z0.d ; CHECK-NEXT: movprfx z24, z3 ; CHECK-NEXT: fcvtzs z24.d, p0/m, z3.d +; CHECK-NEXT: mov z6.d, x8 ; CHECK-NEXT: movprfx z25, z2 ; CHECK-NEXT: fcvtzs z25.d, p0/m, z2.d -; CHECK-NEXT: fcmgt p5.d, p0/z, z1.d, z5.d -; CHECK-NEXT: fcmgt p6.d, p0/z, z0.d, z5.d -; CHECK-NEXT: fcmgt p7.d, p0/z, z3.d, z5.d +; CHECK-NEXT: fcmge p1.d, p0/z, z1.d, z4.d +; CHECK-NEXT: fcmge p2.d, p0/z, z0.d, z4.d +; CHECK-NEXT: fcmge p3.d, p0/z, z3.d, z4.d +; CHECK-NEXT: fcmge p4.d, p0/z, z2.d, z4.d +; CHECK-NEXT: movprfx z4, z1 +; CHECK-NEXT: fcvtzs z4.d, p0/m, z1.d +; CHECK-NEXT: fcmgt p5.d, p0/z, z1.d, z6.d +; CHECK-NEXT: fcmgt p6.d, p0/z, z0.d, z6.d +; CHECK-NEXT: fcmgt p7.d, p0/z, z3.d, z6.d ; CHECK-NEXT: not p1.b, p0/z, p1.b ; CHECK-NEXT: not p2.b, p0/z, p2.b ; CHECK-NEXT: not p3.b, p0/z, p3.b +; CHECK-NEXT: mov z4.d, p1/m, z5.d +; CHECK-NEXT: fcmgt p1.d, p0/z, z2.d, z6.d ; CHECK-NEXT: not p4.b, p0/z, p4.b -; CHECK-NEXT: mov z6.d, p1/m, z4.d -; CHECK-NEXT: fcmgt p1.d, p0/z, z2.d, z5.d -; CHECK-NEXT: sel z5.d, p2, z4.d, z7.d +; CHECK-NEXT: sel z6.d, p2, z5.d, z7.d ; CHECK-NEXT: fcmuo p2.d, p0/z, z1.d, z1.d -; CHECK-NEXT: sel z7.d, p3, z4.d, z24.d +; CHECK-NEXT: sel z7.d, p3, z5.d, z24.d ; CHECK-NEXT: fcmuo p3.d, p0/z, z0.d, z0.d -; CHECK-NEXT: sel z4.d, p4, z4.d, z25.d +; CHECK-NEXT: sel z5.d, p4, z5.d, z25.d ; CHECK-NEXT: fcmuo p4.d, p0/z, z3.d, z3.d ; CHECK-NEXT: fcmuo p0.d, p0/z, z2.d, z2.d -; CHECK-NEXT: sel z0.d, p5, z26.d, z6.d -; CHECK-NEXT: sel z1.d, p6, z26.d, z5.d +; CHECK-NEXT: sel z0.d, p5, z26.d, z4.d +; CHECK-NEXT: sel z1.d, p6, z26.d, z6.d ; CHECK-NEXT: ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: sel z2.d, p7, z26.d, z7.d ; CHECK-NEXT: ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: sel z3.d, p1, z26.d, z4.d +; CHECK-NEXT: sel z3.d, p1, z26.d, z5.d ; CHECK-NEXT: ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: mov z0.d, p2/m, #0 // =0x0 ; CHECK-NEXT: mov z1.d, p3/m, #0 // =0x0 @@ -387,28 +387,28 @@ define @test_signed_v4f64_v4i16( %f) { ; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, #-4548635623644200960 // =0xc0e0000000000000 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z5.d, #32767 // =0x7fff ; CHECK-NEXT: mov z2.d, x8 ; CHECK-NEXT: mov x8, #281200098803712 // =0xffc000000000 ; CHECK-NEXT: movk x8, #16607, lsl #48 -; CHECK-NEXT: mov z3.d, x8 -; CHECK-NEXT: fcmge p1.d, p0/z, z1.d, z2.d -; CHECK-NEXT: fcmge p2.d, p0/z, z0.d, z2.d -; CHECK-NEXT: movprfx z2, z1 -; CHECK-NEXT: fcvtzs z2.d, p0/m, z1.d +; CHECK-NEXT: movprfx z3, z1 +; CHECK-NEXT: fcvtzs z3.d, p0/m, z1.d ; CHECK-NEXT: movprfx z4, z0 ; CHECK-NEXT: fcvtzs z4.d, p0/m, z0.d -; CHECK-NEXT: fcmgt p3.d, p0/z, z1.d, z3.d -; CHECK-NEXT: fcmgt p4.d, p0/z, z0.d, z3.d +; CHECK-NEXT: fcmge p1.d, p0/z, z1.d, z2.d +; CHECK-NEXT: fcmge p2.d, p0/z, z0.d, z2.d +; CHECK-NEXT: mov z2.d, x8 +; CHECK-NEXT: fcmgt p3.d, p0/z, z1.d, z2.d +; CHECK-NEXT: fcmgt p4.d, p0/z, z0.d, z2.d ; CHECK-NEXT: not p1.b, p0/z, p1.b ; CHECK-NEXT: not p2.b, p0/z, p2.b -; CHECK-NEXT: mov z2.d, p1/m, #-32768 // =0xffffffffffff8000 +; CHECK-NEXT: mov z3.d, p1/m, #-32768 // =0xffffffffffff8000 ; CHECK-NEXT: fcmuo p1.d, p0/z, z1.d, z1.d ; CHECK-NEXT: fcmuo p0.d, p0/z, z0.d, z0.d ; CHECK-NEXT: mov z4.d, p2/m, #-32768 // =0xffffffffffff8000 -; CHECK-NEXT: sel z0.d, p3, z5.d, z2.d +; CHECK-NEXT: sel z0.d, p3, z5.d, z3.d ; CHECK-NEXT: sel z1.d, p4, z5.d, z4.d ; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: mov z0.d, p1/m, #0 // =0x0 @@ -432,34 +432,34 @@ define @test_signed_v8f64_v8i16( %f) { ; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, #-4548635623644200960 // =0xc0e0000000000000 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z25.d, #32767 // =0x7fff ; CHECK-NEXT: mov z4.d, x8 ; CHECK-NEXT: mov x8, #281200098803712 // =0xffc000000000 ; CHECK-NEXT: movk x8, #16607, lsl #48 -; CHECK-NEXT: fcmge p1.d, p0/z, z3.d, z4.d -; CHECK-NEXT: fcmge p2.d, p0/z, z2.d, z4.d -; CHECK-NEXT: fcmge p3.d, p0/z, z1.d, z4.d -; CHECK-NEXT: fcmge p4.d, p0/z, z0.d, z4.d -; CHECK-NEXT: movprfx z5, z3 -; CHECK-NEXT: fcvtzs z5.d, p0/m, z3.d -; CHECK-NEXT: mov z4.d, x8 ; CHECK-NEXT: movprfx z6, z2 ; CHECK-NEXT: fcvtzs z6.d, p0/m, z2.d ; CHECK-NEXT: movprfx z7, z1 ; CHECK-NEXT: fcvtzs z7.d, p0/m, z1.d +; CHECK-NEXT: mov z5.d, x8 ; CHECK-NEXT: movprfx z24, z0 ; CHECK-NEXT: fcvtzs z24.d, p0/m, z0.d +; CHECK-NEXT: fcmge p1.d, p0/z, z3.d, z4.d +; CHECK-NEXT: fcmge p2.d, p0/z, z2.d, z4.d +; CHECK-NEXT: fcmge p3.d, p0/z, z1.d, z4.d +; CHECK-NEXT: fcmge p4.d, p0/z, z0.d, z4.d +; CHECK-NEXT: movprfx z4, z3 +; CHECK-NEXT: fcvtzs z4.d, p0/m, z3.d +; CHECK-NEXT: fcmgt p5.d, p0/z, z3.d, z5.d +; CHECK-NEXT: fcmgt p6.d, p0/z, z2.d, z5.d +; CHECK-NEXT: fcmgt p7.d, p0/z, z1.d, z5.d ; CHECK-NEXT: not p1.b, p0/z, p1.b -; CHECK-NEXT: fcmgt p5.d, p0/z, z3.d, z4.d -; CHECK-NEXT: fcmgt p6.d, p0/z, z2.d, z4.d ; CHECK-NEXT: not p2.b, p0/z, p2.b -; CHECK-NEXT: fcmgt p7.d, p0/z, z1.d, z4.d ; CHECK-NEXT: not p3.b, p0/z, p3.b +; CHECK-NEXT: mov z4.d, p1/m, #-32768 // =0xffffffffffff8000 +; CHECK-NEXT: fcmgt p1.d, p0/z, z0.d, z5.d ; CHECK-NEXT: not p4.b, p0/z, p4.b -; CHECK-NEXT: mov z5.d, p1/m, #-32768 // =0xffffffffffff8000 -; CHECK-NEXT: fcmgt p1.d, p0/z, z0.d, z4.d ; CHECK-NEXT: mov z6.d, p2/m, #-32768 // =0xffffffffffff8000 ; CHECK-NEXT: fcmuo p2.d, p0/z, z3.d, z3.d ; CHECK-NEXT: mov z7.d, p3/m, #-32768 // =0xffffffffffff8000 @@ -467,7 +467,7 @@ define @test_signed_v8f64_v8i16( %f) { ; CHECK-NEXT: mov z24.d, p4/m, #-32768 // =0xffffffffffff8000 ; CHECK-NEXT: fcmuo p4.d, p0/z, z1.d, z1.d ; CHECK-NEXT: fcmuo p0.d, p0/z, z0.d, z0.d -; CHECK-NEXT: sel z2.d, p5, z25.d, z5.d +; CHECK-NEXT: sel z2.d, p5, z25.d, z4.d ; CHECK-NEXT: sel z0.d, p6, z25.d, z6.d ; CHECK-NEXT: ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: sel z1.d, p7, z25.d, z7.d @@ -492,8 +492,8 @@ define @test_signed_v8f64_v8i16( %f) { define @test_signed_v2f64_v2i64( %f) { ; CHECK-LABEL: test_signed_v2f64_v2i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, #-4332462841530417152 // =0xc3e0000000000000 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z2.d, #0x8000000000000000 ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: mov x8, #4890909195324358655 // =0x43dfffffffffffff @@ -521,29 +521,29 @@ define @test_signed_v4f64_v4i64( %f) { ; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, #-4332462841530417152 // =0xc3e0000000000000 -; CHECK-NEXT: mov z3.d, #0x8000000000000000 +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: mov z6.d, #0x7fffffffffffffff ; CHECK-NEXT: mov z2.d, x8 ; CHECK-NEXT: mov x8, #4890909195324358655 // =0x43dfffffffffffff -; CHECK-NEXT: mov z6.d, #0x7fffffffffffffff -; CHECK-NEXT: mov z4.d, x8 -; CHECK-NEXT: fcmge p1.d, p0/z, z0.d, z2.d -; CHECK-NEXT: fcmge p2.d, p0/z, z1.d, z2.d -; CHECK-NEXT: movprfx z2, z0 -; CHECK-NEXT: fcvtzs z2.d, p0/m, z0.d +; CHECK-NEXT: mov z3.d, x8 +; CHECK-NEXT: movprfx z4, z0 +; CHECK-NEXT: fcvtzs z4.d, p0/m, z0.d ; CHECK-NEXT: movprfx z5, z1 ; CHECK-NEXT: fcvtzs z5.d, p0/m, z1.d -; CHECK-NEXT: fcmgt p3.d, p0/z, z0.d, z4.d -; CHECK-NEXT: fcmgt p4.d, p0/z, z1.d, z4.d +; CHECK-NEXT: fcmge p1.d, p0/z, z0.d, z2.d +; CHECK-NEXT: fcmge p2.d, p0/z, z1.d, z2.d +; CHECK-NEXT: mov z2.d, #0x8000000000000000 +; CHECK-NEXT: fcmgt p3.d, p0/z, z0.d, z3.d +; CHECK-NEXT: fcmgt p4.d, p0/z, z1.d, z3.d ; CHECK-NEXT: not p1.b, p0/z, p1.b ; CHECK-NEXT: not p2.b, p0/z, p2.b -; CHECK-NEXT: mov z2.d, p1/m, z3.d +; CHECK-NEXT: sel z3.d, p1, z2.d, z4.d ; CHECK-NEXT: fcmuo p1.d, p0/z, z0.d, z0.d ; CHECK-NEXT: fcmuo p0.d, p0/z, z1.d, z1.d -; CHECK-NEXT: sel z3.d, p2, z3.d, z5.d -; CHECK-NEXT: sel z0.d, p3, z6.d, z2.d -; CHECK-NEXT: sel z1.d, p4, z6.d, z3.d +; CHECK-NEXT: sel z2.d, p2, z2.d, z5.d +; CHECK-NEXT: sel z0.d, p3, z6.d, z3.d +; CHECK-NEXT: sel z1.d, p4, z6.d, z2.d ; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: mov z0.d, p1/m, #0 // =0x0 ; CHECK-NEXT: mov z1.d, p0/m, #0 // =0x0 @@ -568,8 +568,8 @@ declare @llvm.fptosi.sat.nxv4f16.nxv4i64() define @test_signed_v2f16_v2i32( %f) { ; CHECK-LABEL: test_signed_v2f16_v2i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, #64511 // =0xfbff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z2.d, #0xffffffff80000000 ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: mov w8, #31743 // =0x7bff @@ -592,8 +592,8 @@ define @test_signed_v2f16_v2i32( %f) { define @test_signed_v4f16_v4i32( %f) { ; CHECK-LABEL: test_signed_v4f16_v4i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #64511 // =0xfbff +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, #0x80000000 ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: mov w8, #31743 // =0x7bff @@ -621,31 +621,31 @@ define @test_signed_v8f16_v8i32( %f) { ; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #64511 // =0xfbff ; CHECK-NEXT: uunpklo z1.s, z0.h ; CHECK-NEXT: uunpkhi z0.s, z0.h ; CHECK-NEXT: mov z2.h, w8 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #31743 // =0x7bff -; CHECK-NEXT: mov z3.s, #0x80000000 -; CHECK-NEXT: mov z4.h, w8 +; CHECK-NEXT: mov z3.h, w8 ; CHECK-NEXT: mov z6.s, #0x7fffffff ; CHECK-NEXT: fcmge p1.h, p0/z, z1.h, z2.h ; CHECK-NEXT: fcmge p2.h, p0/z, z0.h, z2.h -; CHECK-NEXT: movprfx z2, z1 -; CHECK-NEXT: fcvtzs z2.s, p0/m, z1.h +; CHECK-NEXT: mov z2.s, #0x80000000 +; CHECK-NEXT: movprfx z4, z1 +; CHECK-NEXT: fcvtzs z4.s, p0/m, z1.h ; CHECK-NEXT: movprfx z5, z0 ; CHECK-NEXT: fcvtzs z5.s, p0/m, z0.h -; CHECK-NEXT: fcmgt p3.h, p0/z, z1.h, z4.h -; CHECK-NEXT: fcmgt p4.h, p0/z, z0.h, z4.h +; CHECK-NEXT: fcmgt p3.h, p0/z, z1.h, z3.h +; CHECK-NEXT: fcmgt p4.h, p0/z, z0.h, z3.h ; CHECK-NEXT: not p1.b, p0/z, p1.b ; CHECK-NEXT: not p2.b, p0/z, p2.b -; CHECK-NEXT: mov z2.s, p1/m, z3.s +; CHECK-NEXT: sel z3.s, p1, z2.s, z4.s ; CHECK-NEXT: fcmuo p1.h, p0/z, z1.h, z1.h ; CHECK-NEXT: fcmuo p0.h, p0/z, z0.h, z0.h -; CHECK-NEXT: sel z3.s, p2, z3.s, z5.s -; CHECK-NEXT: sel z0.s, p3, z6.s, z2.s -; CHECK-NEXT: sel z1.s, p4, z6.s, z3.s +; CHECK-NEXT: sel z2.s, p2, z2.s, z5.s +; CHECK-NEXT: sel z0.s, p3, z6.s, z3.s +; CHECK-NEXT: sel z1.s, p4, z6.s, z2.s ; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: mov z0.s, p1/m, #0 // =0x0 ; CHECK-NEXT: mov z1.s, p0/m, #0 // =0x0 @@ -659,8 +659,8 @@ define @test_signed_v8f16_v8i32( %f) { define @test_signed_v4f16_v4i16( %f) { ; CHECK-LABEL: test_signed_v4f16_v4i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #63488 // =0xf800 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: mov w8, #30719 // =0x77ff ; CHECK-NEXT: mov z2.h, w8 @@ -682,8 +682,8 @@ define @test_signed_v4f16_v4i16( %f) { define @test_signed_v8f16_v8i16( %f) { ; CHECK-LABEL: test_signed_v8f16_v8i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w8, #63488 // =0xf800 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: mov w8, #30719 // =0x77ff ; CHECK-NEXT: mov z2.h, w8 @@ -705,8 +705,8 @@ define @test_signed_v8f16_v8i16( %f) { define @test_signed_v2f16_v2i64( %f) { ; CHECK-LABEL: test_signed_v2f16_v2i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, #64511 // =0xfbff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z2.d, #0x8000000000000000 ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: mov w8, #31743 // =0x7bff @@ -734,31 +734,31 @@ define @test_signed_v4f16_v4i64( %f) { ; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, #64511 // =0xfbff ; CHECK-NEXT: uunpklo z1.d, z0.s ; CHECK-NEXT: uunpkhi z0.d, z0.s ; CHECK-NEXT: mov z2.h, w8 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, #31743 // =0x7bff -; CHECK-NEXT: mov z3.d, #0x8000000000000000 -; CHECK-NEXT: mov z4.h, w8 +; CHECK-NEXT: mov z3.h, w8 ; CHECK-NEXT: mov z6.d, #0x7fffffffffffffff ; CHECK-NEXT: fcmge p1.h, p0/z, z1.h, z2.h ; CHECK-NEXT: fcmge p2.h, p0/z, z0.h, z2.h -; CHECK-NEXT: movprfx z2, z1 -; CHECK-NEXT: fcvtzs z2.d, p0/m, z1.h +; CHECK-NEXT: mov z2.d, #0x8000000000000000 +; CHECK-NEXT: movprfx z4, z1 +; CHECK-NEXT: fcvtzs z4.d, p0/m, z1.h ; CHECK-NEXT: movprfx z5, z0 ; CHECK-NEXT: fcvtzs z5.d, p0/m, z0.h -; CHECK-NEXT: fcmgt p3.h, p0/z, z1.h, z4.h -; CHECK-NEXT: fcmgt p4.h, p0/z, z0.h, z4.h +; CHECK-NEXT: fcmgt p3.h, p0/z, z1.h, z3.h +; CHECK-NEXT: fcmgt p4.h, p0/z, z0.h, z3.h ; CHECK-NEXT: not p1.b, p0/z, p1.b ; CHECK-NEXT: not p2.b, p0/z, p2.b -; CHECK-NEXT: mov z2.d, p1/m, z3.d +; CHECK-NEXT: sel z3.d, p1, z2.d, z4.d ; CHECK-NEXT: fcmuo p1.h, p0/z, z1.h, z1.h ; CHECK-NEXT: fcmuo p0.h, p0/z, z0.h, z0.h -; CHECK-NEXT: sel z3.d, p2, z3.d, z5.d -; CHECK-NEXT: sel z0.d, p3, z6.d, z2.d -; CHECK-NEXT: sel z1.d, p4, z6.d, z3.d +; CHECK-NEXT: sel z2.d, p2, z2.d, z5.d +; CHECK-NEXT: sel z0.d, p3, z6.d, z3.d +; CHECK-NEXT: sel z1.d, p4, z6.d, z2.d ; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: mov z0.d, p1/m, #0 // =0x0 ; CHECK-NEXT: mov z1.d, p0/m, #0 // =0x0 diff --git a/llvm/test/CodeGen/AArch64/sve-fptoui-sat.ll b/llvm/test/CodeGen/AArch64/sve-fptoui-sat.ll index c56c0b37888d..ed352ffec339 100644 --- a/llvm/test/CodeGen/AArch64/sve-fptoui-sat.ll +++ b/llvm/test/CodeGen/AArch64/sve-fptoui-sat.ll @@ -82,8 +82,8 @@ define @test_signed_v4f32_v4i16( %f) { ; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #65280 // =0xff00 ; CHECK-NEXT: movk w8, #18303, lsl #16 -; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: fcmge p1.s, p0/z, z0.s, #0.0 +; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: movprfx z2, z0 ; CHECK-NEXT: fcvtzu z2.s, p0/m, z0.s ; CHECK-NEXT: not p1.b, p0/z, p1.b @@ -102,9 +102,9 @@ define @test_signed_v8f32_v8i16( %f) { ; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #65280 // =0xff00 ; CHECK-NEXT: movk w8, #18303, lsl #16 -; CHECK-NEXT: mov z2.s, w8 ; CHECK-NEXT: fcmge p1.s, p0/z, z1.s, #0.0 ; CHECK-NEXT: fcmge p2.s, p0/z, z0.s, #0.0 +; CHECK-NEXT: mov z2.s, w8 ; CHECK-NEXT: movprfx z3, z1 ; CHECK-NEXT: fcvtzu z3.s, p0/m, z1.s ; CHECK-NEXT: movprfx z4, z0 @@ -146,10 +146,10 @@ define @test_signed_v2f32_v2i64( %f) { define @test_signed_v4f32_v4i64( %f) { ; CHECK-LABEL: test_signed_v4f32_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: uunpklo z2.d, z0.s ; CHECK-NEXT: uunpkhi z3.d, z0.s ; CHECK-NEXT: mov w8, #1602224127 // =0x5f7fffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z4.s, w8 ; CHECK-NEXT: fcmge p1.s, p0/z, z2.s, #0.0 ; CHECK-NEXT: fcmge p2.s, p0/z, z3.s, #0.0 @@ -186,8 +186,8 @@ define @test_signed_v2f64_v2i32( %f) { ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, #281474974613504 // =0xffffffe00000 ; CHECK-NEXT: movk x8, #16879, lsl #48 -; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: fcmge p1.d, p0/z, z0.d, #0.0 +; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: movprfx z2, z0 ; CHECK-NEXT: fcvtzu z2.d, p0/m, z0.d ; CHECK-NEXT: not p1.b, p0/z, p1.b @@ -206,9 +206,9 @@ define @test_signed_v4f64_v4i32( %f) { ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, #281474974613504 // =0xffffffe00000 ; CHECK-NEXT: movk x8, #16879, lsl #48 -; CHECK-NEXT: mov z2.d, x8 ; CHECK-NEXT: fcmge p1.d, p0/z, z1.d, #0.0 ; CHECK-NEXT: fcmge p2.d, p0/z, z0.d, #0.0 +; CHECK-NEXT: mov z2.d, x8 ; CHECK-NEXT: movprfx z3, z1 ; CHECK-NEXT: fcvtzu z3.d, p0/m, z1.d ; CHECK-NEXT: movprfx z4, z0 @@ -241,28 +241,28 @@ define @test_signed_v8f64_v8i32( %f) { ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, #281474974613504 // =0xffffffe00000 ; CHECK-NEXT: movk x8, #16879, lsl #48 -; CHECK-NEXT: mov z4.d, x8 ; CHECK-NEXT: fcmge p1.d, p0/z, z1.d, #0.0 ; CHECK-NEXT: fcmge p2.d, p0/z, z0.d, #0.0 ; CHECK-NEXT: fcmge p3.d, p0/z, z3.d, #0.0 ; CHECK-NEXT: fcmge p4.d, p0/z, z2.d, #0.0 ; CHECK-NEXT: movprfx z5, z1 ; CHECK-NEXT: fcvtzu z5.d, p0/m, z1.d +; CHECK-NEXT: mov z4.d, x8 ; CHECK-NEXT: movprfx z6, z0 ; CHECK-NEXT: fcvtzu z6.d, p0/m, z0.d ; CHECK-NEXT: movprfx z7, z3 ; CHECK-NEXT: fcvtzu z7.d, p0/m, z3.d ; CHECK-NEXT: movprfx z24, z2 ; CHECK-NEXT: fcvtzu z24.d, p0/m, z2.d +; CHECK-NEXT: not p1.b, p0/z, p1.b ; CHECK-NEXT: fcmgt p5.d, p0/z, z1.d, z4.d ; CHECK-NEXT: fcmgt p6.d, p0/z, z0.d, z4.d -; CHECK-NEXT: mov z0.d, #0xffffffff -; CHECK-NEXT: not p1.b, p0/z, p1.b ; CHECK-NEXT: not p2.b, p0/z, p2.b +; CHECK-NEXT: mov z0.d, #0xffffffff ; CHECK-NEXT: not p3.b, p0/z, p3.b -; CHECK-NEXT: not p4.b, p0/z, p4.b ; CHECK-NEXT: mov z5.d, p1/m, #0 // =0x0 ; CHECK-NEXT: fcmgt p1.d, p0/z, z3.d, z4.d +; CHECK-NEXT: not p4.b, p0/z, p4.b ; CHECK-NEXT: fcmgt p0.d, p0/z, z2.d, z4.d ; CHECK-NEXT: mov z6.d, p2/m, #0 // =0x0 ; CHECK-NEXT: mov z7.d, p3/m, #0 // =0x0 @@ -289,9 +289,9 @@ define @test_signed_v4f64_v4i16( %f) { ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, #281337537757184 // =0xffe000000000 ; CHECK-NEXT: movk x8, #16623, lsl #48 -; CHECK-NEXT: mov z2.d, x8 ; CHECK-NEXT: fcmge p1.d, p0/z, z1.d, #0.0 ; CHECK-NEXT: fcmge p2.d, p0/z, z0.d, #0.0 +; CHECK-NEXT: mov z2.d, x8 ; CHECK-NEXT: movprfx z3, z1 ; CHECK-NEXT: fcvtzu z3.d, p0/m, z1.d ; CHECK-NEXT: movprfx z4, z0 @@ -324,28 +324,28 @@ define @test_signed_v8f64_v8i16( %f) { ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, #281337537757184 // =0xffe000000000 ; CHECK-NEXT: movk x8, #16623, lsl #48 -; CHECK-NEXT: mov z4.d, x8 ; CHECK-NEXT: fcmge p1.d, p0/z, z3.d, #0.0 ; CHECK-NEXT: fcmge p2.d, p0/z, z2.d, #0.0 ; CHECK-NEXT: fcmge p3.d, p0/z, z1.d, #0.0 ; CHECK-NEXT: fcmge p4.d, p0/z, z0.d, #0.0 ; CHECK-NEXT: movprfx z5, z3 ; CHECK-NEXT: fcvtzu z5.d, p0/m, z3.d +; CHECK-NEXT: mov z4.d, x8 ; CHECK-NEXT: movprfx z6, z2 ; CHECK-NEXT: fcvtzu z6.d, p0/m, z2.d ; CHECK-NEXT: movprfx z7, z1 ; CHECK-NEXT: fcvtzu z7.d, p0/m, z1.d ; CHECK-NEXT: movprfx z24, z0 ; CHECK-NEXT: fcvtzu z24.d, p0/m, z0.d +; CHECK-NEXT: not p1.b, p0/z, p1.b ; CHECK-NEXT: fcmgt p5.d, p0/z, z3.d, z4.d ; CHECK-NEXT: fcmgt p6.d, p0/z, z2.d, z4.d -; CHECK-NEXT: mov z2.d, #65535 // =0xffff -; CHECK-NEXT: not p1.b, p0/z, p1.b ; CHECK-NEXT: not p2.b, p0/z, p2.b +; CHECK-NEXT: mov z2.d, #65535 // =0xffff ; CHECK-NEXT: not p3.b, p0/z, p3.b -; CHECK-NEXT: not p4.b, p0/z, p4.b ; CHECK-NEXT: mov z5.d, p1/m, #0 // =0x0 ; CHECK-NEXT: fcmgt p1.d, p0/z, z1.d, z4.d +; CHECK-NEXT: not p4.b, p0/z, p4.b ; CHECK-NEXT: fcmgt p0.d, p0/z, z0.d, z4.d ; CHECK-NEXT: mov z6.d, p2/m, #0 // =0x0 ; CHECK-NEXT: mov z7.d, p3/m, #0 // =0x0 @@ -465,10 +465,10 @@ define @test_signed_v4f16_v4i32( %f) { define @test_signed_v8f16_v8i32( %f) { ; CHECK-LABEL: test_signed_v8f16_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uunpklo z2.s, z0.h ; CHECK-NEXT: uunpkhi z3.s, z0.h ; CHECK-NEXT: mov w8, #31743 // =0x7bff +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z4.h, w8 ; CHECK-NEXT: fcmge p1.h, p0/z, z2.h, #0.0 ; CHECK-NEXT: fcmge p2.h, p0/z, z3.h, #0.0 @@ -549,10 +549,10 @@ define @test_signed_v2f16_v2i64( %f) { define @test_signed_v4f16_v4i64( %f) { ; CHECK-LABEL: test_signed_v4f16_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: uunpklo z2.d, z0.s ; CHECK-NEXT: uunpkhi z3.d, z0.s ; CHECK-NEXT: mov w8, #31743 // =0x7bff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z4.h, w8 ; CHECK-NEXT: fcmge p1.h, p0/z, z2.h, #0.0 ; CHECK-NEXT: fcmge p2.h, p0/z, z3.h, #0.0 diff --git a/llvm/test/CodeGen/AArch64/sve-gather-scatter-addr-opts.ll b/llvm/test/CodeGen/AArch64/sve-gather-scatter-addr-opts.ll index 5c4c9463528b..ad6371f78ec0 100644 --- a/llvm/test/CodeGen/AArch64/sve-gather-scatter-addr-opts.ll +++ b/llvm/test/CodeGen/AArch64/sve-gather-scatter-addr-opts.ll @@ -71,12 +71,12 @@ define @gather_i8_index_offset_8(ptr %base, i64 %offset, %pg, %data) #0 { ; CHECK-LABEL: scatter_f16_index_offset_var: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: index z1.d, #0, #1 ; CHECK-NEXT: mov z2.d, x1 -; CHECK-NEXT: punpklo p2.h, p0.b +; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: uunpklo z3.d, z0.s ; CHECK-NEXT: uunpkhi z0.d, z0.s +; CHECK-NEXT: punpklo p2.h, p0.b ; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: movprfx z4, z2 ; CHECK-NEXT: mla z4.d, p1/m, z1.d, z2.d @@ -101,16 +101,16 @@ define void @scatter_f16_index_offset_var(ptr %base, i64 %offset, i64 %scale, %pg, %data) #0 { ; CHECK-LABEL: scatter_i8_index_offset_maximum_plus_one: ; CHECK: // %bb.0: -; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: mov w8, #33554432 // =0x2000000 ; CHECK-NEXT: uunpklo z2.d, z0.s -; CHECK-NEXT: index z1.d, #0, x8 ; CHECK-NEXT: rdvl x9, #1 -; CHECK-NEXT: add x8, x0, x1 +; CHECK-NEXT: index z1.d, #0, x8 +; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: lsr x9, x9, #4 +; CHECK-NEXT: add x8, x0, x1 ; CHECK-NEXT: mov w10, #67108864 // =0x4000000 -; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: uunpkhi z0.d, z0.s +; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: st1b { z2.d }, p1, [x8, z1.d] ; CHECK-NEXT: madd x8, x9, x10, x8 ; CHECK-NEXT: st1b { z0.d }, p0, [x8, z1.d] @@ -131,17 +131,17 @@ define void @scatter_i8_index_offset_maximum_plus_one(ptr %base, i64 %offset, %pg, %data) #0 { ; CHECK-LABEL: scatter_i8_index_offset_minimum_minus_one: ; CHECK: // %bb.0: -; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: mov x8, #-33554433 // =0xfffffffffdffffff ; CHECK-NEXT: uunpklo z2.d, z0.s -; CHECK-NEXT: index z1.d, #0, x8 ; CHECK-NEXT: rdvl x9, #1 -; CHECK-NEXT: mov x10, #-2 // =0xfffffffffffffffe +; CHECK-NEXT: index z1.d, #0, x8 +; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: lsr x9, x9, #4 +; CHECK-NEXT: mov x10, #-2 // =0xfffffffffffffffe ; CHECK-NEXT: add x8, x0, x1 +; CHECK-NEXT: uunpkhi z0.d, z0.s ; CHECK-NEXT: movk x10, #64511, lsl #16 ; CHECK-NEXT: punpkhi p0.h, p0.b -; CHECK-NEXT: uunpkhi z0.d, z0.s ; CHECK-NEXT: st1b { z2.d }, p1, [x8, z1.d] ; CHECK-NEXT: madd x8, x9, x10, x8 ; CHECK-NEXT: st1b { z0.d }, p0, [x8, z1.d] @@ -162,16 +162,16 @@ define void @scatter_i8_index_offset_minimum_minus_one(ptr %base, i64 %offset, < define void @scatter_i8_index_stride_too_big(ptr %base, i64 %offset, %pg, %data) #0 { ; CHECK-LABEL: scatter_i8_index_stride_too_big: ; CHECK: // %bb.0: -; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: mov x8, #4611686018427387904 // =0x4000000000000000 ; CHECK-NEXT: uunpklo z2.d, z0.s -; CHECK-NEXT: index z1.d, #0, x8 ; CHECK-NEXT: rdvl x9, #1 -; CHECK-NEXT: add x8, x0, x1 +; CHECK-NEXT: index z1.d, #0, x8 +; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: lsr x9, x9, #4 +; CHECK-NEXT: add x8, x0, x1 ; CHECK-NEXT: mov x10, #-9223372036854775808 // =0x8000000000000000 -; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: uunpkhi z0.d, z0.s +; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: st1b { z2.d }, p1, [x8, z1.d] ; CHECK-NEXT: madd x8, x9, x10, x8 ; CHECK-NEXT: st1b { z0.d }, p0, [x8, z1.d] diff --git a/llvm/test/CodeGen/AArch64/sve-hadd.ll b/llvm/test/CodeGen/AArch64/sve-hadd.ll index c73370d50287..f90aef8daa5d 100644 --- a/llvm/test/CodeGen/AArch64/sve-hadd.ll +++ b/llvm/test/CodeGen/AArch64/sve-hadd.ll @@ -129,9 +129,9 @@ define @haddu_v2i32( %s0, @haddu_v2i16( %s0, @haddu_v4i16( %s0, @haddu_v4i8( %s0, %s ; ; SVE2-LABEL: haddu_v4i8: ; SVE2: // %bb.0: // %entry -; SVE2-NEXT: ptrue p0.s ; SVE2-NEXT: and z0.s, z0.s, #0xff ; SVE2-NEXT: and z1.s, z1.s, #0xff +; SVE2-NEXT: ptrue p0.s ; SVE2-NEXT: uhadd z0.s, p0/m, z0.s, z1.s ; SVE2-NEXT: ret entry: @@ -557,9 +557,9 @@ define @haddu_v8i8( %s0, %s ; ; SVE2-LABEL: haddu_v8i8: ; SVE2: // %bb.0: // %entry -; SVE2-NEXT: ptrue p0.h ; SVE2-NEXT: and z0.h, z0.h, #0xff ; SVE2-NEXT: and z1.h, z1.h, #0xff +; SVE2-NEXT: ptrue p0.h ; SVE2-NEXT: uhadd z0.h, p0/m, z0.h, z1.h ; SVE2-NEXT: ret entry: @@ -787,9 +787,9 @@ define @rhaddu_v2i32( %s0, @rhaddu_v2i16( %s0, @rhaddu_v4i16( %s0, @rhaddu_v4i8( %s0, % ; ; SVE2-LABEL: rhaddu_v4i8: ; SVE2: // %bb.0: // %entry -; SVE2-NEXT: ptrue p0.s ; SVE2-NEXT: and z0.s, z0.s, #0xff ; SVE2-NEXT: and z1.s, z1.s, #0xff +; SVE2-NEXT: ptrue p0.s ; SVE2-NEXT: urhadd z0.s, p0/m, z0.s, z1.s ; SVE2-NEXT: ret entry: @@ -1241,9 +1241,9 @@ define @rhaddu_v8i8( %s0, % ; ; SVE2-LABEL: rhaddu_v8i8: ; SVE2: // %bb.0: // %entry -; SVE2-NEXT: ptrue p0.h ; SVE2-NEXT: and z0.h, z0.h, #0xff ; SVE2-NEXT: and z1.h, z1.h, #0xff +; SVE2-NEXT: ptrue p0.h ; SVE2-NEXT: urhadd z0.h, p0/m, z0.h, z1.h ; SVE2-NEXT: ret entry: diff --git a/llvm/test/CodeGen/AArch64/sve-implicit-zero-filling.ll b/llvm/test/CodeGen/AArch64/sve-implicit-zero-filling.ll index e20399de70bf..73bbee094827 100644 --- a/llvm/test/CodeGen/AArch64/sve-implicit-zero-filling.ll +++ b/llvm/test/CodeGen/AArch64/sve-implicit-zero-filling.ll @@ -175,14 +175,14 @@ define @uminv_zero_fill( %pg, @zero_fill_non_zero_index( %pg, %a) #0 { ; CHECK-LABEL: zero_fill_non_zero_index: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: mov w8, #1 // =0x1 ; CHECK-NEXT: index z1.d, #0, #1 ; CHECK-NEXT: uminv d3, p0, z0.d ; CHECK-NEXT: mov z2.d, x8 +; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: mov z0.d, #0 // =0x0 -; CHECK-NEXT: fmov x8, d3 ; CHECK-NEXT: cmpeq p0.d, p1/z, z1.d, z2.d +; CHECK-NEXT: fmov x8, d3 ; CHECK-NEXT: mov z0.d, p0/m, x8 ; CHECK-NEXT: ret %t1 = call i64 @llvm.aarch64.sve.uminv.nxv2i64( %pg, %a) @@ -210,11 +210,11 @@ define @zero_fill_type_mismatch( %pg, @zero_fill_no_zero_upper_lanes( %pg, %a) #0 { ; CHECK-LABEL: zero_fill_no_zero_upper_lanes: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.d, vl1 ; CHECK-NEXT: umin z0.d, p0/m, z0.d, z0.d ; CHECK-NEXT: mov z1.d, #0 // =0x0 +; CHECK-NEXT: ptrue p0.d, vl1 ; CHECK-NEXT: fmov x8, d0 -; CHECK-NEXT: mov z1.d, p1/m, x8 +; CHECK-NEXT: mov z1.d, p0/m, x8 ; CHECK-NEXT: mov z0.d, z1.d ; CHECK-NEXT: ret %t1 = call @llvm.aarch64.sve.umin.nxv2i64( %pg, %a, %a) diff --git a/llvm/test/CodeGen/AArch64/sve-insert-element.ll b/llvm/test/CodeGen/AArch64/sve-insert-element.ll index 2aa298f6d917..7344964f13bb 100644 --- a/llvm/test/CodeGen/AArch64/sve-insert-element.ll +++ b/llvm/test/CodeGen/AArch64/sve-insert-element.ll @@ -48,8 +48,8 @@ define @test_lane0_2xi64( %a) { define @test_lane0_2xf64( %a) { ; CHECK-LABEL: test_lane0_2xf64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl1 ; CHECK-NEXT: fmov d1, #1.00000000 +; CHECK-NEXT: ptrue p0.d, vl1 ; CHECK-NEXT: mov z0.d, p0/m, z1.d ; CHECK-NEXT: ret %b = insertelement %a, double 1.0, i32 0 @@ -59,8 +59,8 @@ define @test_lane0_2xf64( %a) { define @test_lane0_4xf32( %a) { ; CHECK-LABEL: test_lane0_4xf32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl1 ; CHECK-NEXT: fmov s1, #1.00000000 +; CHECK-NEXT: ptrue p0.s, vl1 ; CHECK-NEXT: mov z0.s, p0/m, z1.s ; CHECK-NEXT: ret %b = insertelement %a, float 1.0, i32 0 @@ -70,8 +70,8 @@ define @test_lane0_4xf32( %a) { define @test_lane0_8xf16( %a) { ; CHECK-LABEL: test_lane0_8xf16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl1 ; CHECK-NEXT: fmov h1, #1.00000000 +; CHECK-NEXT: ptrue p0.h, vl1 ; CHECK-NEXT: mov z0.h, p0/m, z1.h ; CHECK-NEXT: ret %b = insertelement %a, half 1.0, i32 0 @@ -93,9 +93,9 @@ define @test_lane0_8xbf16( %a, bfloat define @test_lane4_2xi64( %a) { ; CHECK-LABEL: test_lane4_2xi64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, #4 // =0x4 ; CHECK-NEXT: index z1.d, #0, #1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z2.d, x8 ; CHECK-NEXT: mov w8, #30 // =0x1e ; CHECK-NEXT: cmpeq p0.d, p0/z, z1.d, z2.d @@ -109,9 +109,9 @@ define @test_lane4_2xi64( %a) { define @test_lane9_8xf16( %a) { ; CHECK-LABEL: test_lane9_8xf16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w8, #9 // =0x9 ; CHECK-NEXT: index z1.h, #0, #1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w8 ; CHECK-NEXT: cmpeq p0.h, p0/z, z1.h, z2.h ; CHECK-NEXT: fmov h1, #1.00000000 @@ -124,9 +124,9 @@ define @test_lane9_8xf16( %a) { define @test_lane9_8xbf16( %a, bfloat %x) { ; CHECK-LABEL: test_lane9_8xbf16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w8, #9 // =0x9 ; CHECK-NEXT: index z2.h, #0, #1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z3.h, w8 ; CHECK-NEXT: cmpeq p0.h, p0/z, z2.h, z3.h ; CHECK-NEXT: mov z0.h, p0/m, h1 @@ -138,9 +138,9 @@ define @test_lane9_8xbf16( %a, bfloat define @test_lane1_16xi8( %a) { ; CHECK-LABEL: test_lane1_16xi8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov w8, #1 // =0x1 ; CHECK-NEXT: index z1.b, #0, #1 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z2.b, w8 ; CHECK-NEXT: mov w8, #30 // =0x1e ; CHECK-NEXT: cmpeq p0.b, p0/z, z1.b, z2.b @@ -153,9 +153,9 @@ define @test_lane1_16xi8( %a) { define @test_lanex_16xi8( %a, i32 %x) { ; CHECK-LABEL: test_lanex_16xi8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b -; CHECK-NEXT: mov w8, w0 ; CHECK-NEXT: index z1.b, #0, #1 +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z2.b, w8 ; CHECK-NEXT: mov w8, #30 // =0x1e ; CHECK-NEXT: cmpeq p0.b, p0/z, z1.b, z2.b @@ -179,9 +179,9 @@ define @extract_insert_4xi32( %a) { define @test_lane6_undef_8xi16(i16 %a) { ; CHECK-LABEL: test_lane6_undef_8xi16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w8, #6 // =0x6 ; CHECK-NEXT: index z0.h, #0, #1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: cmpeq p0.h, p0/z, z0.h, z1.h ; CHECK-NEXT: mov z0.h, p0/m, w0 @@ -202,8 +202,8 @@ define @test_lane0_undef_16xi8(i8 %a) { define @test_insert0_of_extract0_16xi8( %a, %b) { ; CHECK-LABEL: test_insert0_of_extract0_16xi8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl1 ; CHECK-NEXT: fmov w8, s1 +; CHECK-NEXT: ptrue p0.b, vl1 ; CHECK-NEXT: mov z0.b, p0/m, w8 ; CHECK-NEXT: ret %c = extractelement %b, i32 0 @@ -215,12 +215,12 @@ define @test_insert64_of_extract64_16xi8( % ; CHECK-LABEL: test_insert64_of_extract64_16xi8: ; CHECK: // %bb.0: ; CHECK-NEXT: mov w8, #64 // =0x40 -; CHECK-NEXT: ptrue p1.b ; CHECK-NEXT: whilels p0.b, xzr, x8 ; CHECK-NEXT: mov z2.b, w8 ; CHECK-NEXT: lastb w9, p0, z1.b ; CHECK-NEXT: index z1.b, #0, #1 -; CHECK-NEXT: cmpeq p0.b, p1/z, z1.b, z2.b +; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: cmpeq p0.b, p0/z, z1.b, z2.b ; CHECK-NEXT: mov z0.b, p0/m, w9 ; CHECK-NEXT: ret %c = extractelement %b, i32 64 @@ -231,9 +231,9 @@ define @test_insert64_of_extract64_16xi8( % define @test_insert3_of_extract1_16xi8( %a, %b) { ; CHECK-LABEL: test_insert3_of_extract1_16xi8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov w8, #3 // =0x3 ; CHECK-NEXT: index z2.b, #0, #1 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z3.b, w8 ; CHECK-NEXT: umov w8, v1.b[1] ; CHECK-NEXT: cmpeq p0.b, p0/z, z2.b, z3.b @@ -329,9 +329,9 @@ define @test_insert_into_undef_nxv2f64(double %a) { define @test_insert_with_index_nxv2f16(half %h, i64 %idx) { ; CHECK-LABEL: test_insert_with_index_nxv2f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: index z1.d, #0, #1 ; CHECK-NEXT: mov z2.d, x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: cmpeq p0.d, p0/z, z1.d, z2.d ; CHECK-NEXT: mov z0.h, p0/m, h0 ; CHECK-NEXT: ret @@ -342,9 +342,9 @@ define @test_insert_with_index_nxv2f16(half %h, i64 %idx) { define @test_insert_with_index_nxv4f16(half %h, i64 %idx) { ; CHECK-LABEL: test_insert_with_index_nxv4f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: index z1.s, #0, #1 ; CHECK-NEXT: mov z2.s, w0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: cmpeq p0.s, p0/z, z1.s, z2.s ; CHECK-NEXT: mov z0.h, p0/m, h0 ; CHECK-NEXT: ret @@ -355,9 +355,9 @@ define @test_insert_with_index_nxv4f16(half %h, i64 %idx) { define @test_insert_with_index_nxv8f16(half %h, i64 %idx) { ; CHECK-LABEL: test_insert_with_index_nxv8f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: index z1.h, #0, #1 ; CHECK-NEXT: mov z2.h, w0 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: cmpeq p0.h, p0/z, z1.h, z2.h ; CHECK-NEXT: mov z0.h, p0/m, h0 ; CHECK-NEXT: ret @@ -368,9 +368,9 @@ define @test_insert_with_index_nxv8f16(half %h, i64 %idx) { define @test_insert_with_index_nxv2bf16(bfloat %h, i64 %idx) { ; CHECK-LABEL: test_insert_with_index_nxv2bf16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: index z1.d, #0, #1 ; CHECK-NEXT: mov z2.d, x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: cmpeq p0.d, p0/z, z1.d, z2.d ; CHECK-NEXT: mov z0.h, p0/m, h0 ; CHECK-NEXT: ret @@ -381,9 +381,9 @@ define @test_insert_with_index_nxv2bf16(bfloat %h, i64 %id define @test_insert_with_index_nxv4bf16(bfloat %h, i64 %idx) { ; CHECK-LABEL: test_insert_with_index_nxv4bf16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: index z1.s, #0, #1 ; CHECK-NEXT: mov z2.s, w0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: cmpeq p0.s, p0/z, z1.s, z2.s ; CHECK-NEXT: mov z0.h, p0/m, h0 ; CHECK-NEXT: ret @@ -394,9 +394,9 @@ define @test_insert_with_index_nxv4bf16(bfloat %h, i64 %id define @test_insert_with_index_nxv8bf16(bfloat %h, i64 %idx) { ; CHECK-LABEL: test_insert_with_index_nxv8bf16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: index z1.h, #0, #1 ; CHECK-NEXT: mov z2.h, w0 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: cmpeq p0.h, p0/z, z1.h, z2.h ; CHECK-NEXT: mov z0.h, p0/m, h0 ; CHECK-NEXT: ret @@ -407,9 +407,9 @@ define @test_insert_with_index_nxv8bf16(bfloat %h, i64 %id define @test_insert_with_index_nxv2f32(float %f, i64 %idx) { ; CHECK-LABEL: test_insert_with_index_nxv2f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: index z1.d, #0, #1 ; CHECK-NEXT: mov z2.d, x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: cmpeq p0.d, p0/z, z1.d, z2.d ; CHECK-NEXT: mov z0.s, p0/m, s0 ; CHECK-NEXT: ret @@ -420,9 +420,9 @@ define @test_insert_with_index_nxv2f32(float %f, i64 %idx) define @test_insert_with_index_nxv4f32(float %f, i64 %idx) { ; CHECK-LABEL: test_insert_with_index_nxv4f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: index z1.s, #0, #1 ; CHECK-NEXT: mov z2.s, w0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: cmpeq p0.s, p0/z, z1.s, z2.s ; CHECK-NEXT: mov z0.s, p0/m, s0 ; CHECK-NEXT: ret @@ -433,9 +433,9 @@ define @test_insert_with_index_nxv4f32(float %f, i64 %idx) define @test_insert_with_index_nxv2f64(double %d, i64 %idx) { ; CHECK-LABEL: test_insert_with_index_nxv2f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: index z1.d, #0, #1 ; CHECK-NEXT: mov z2.d, x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: cmpeq p0.d, p0/z, z1.d, z2.d ; CHECK-NEXT: mov z0.d, p0/m, d0 ; CHECK-NEXT: ret @@ -447,11 +447,11 @@ define @test_insert_with_index_nxv2f64(double %d, i64 %idx define @test_predicate_insert_2xi1_immediate ( %val, i1 %elt) { ; CHECK-LABEL: test_predicate_insert_2xi1_immediate: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.d, vl1 ; CHECK-NEXT: mov z0.d, p0/z, #1 // =0x1 +; CHECK-NEXT: ptrue p0.d, vl1 ; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 +; CHECK-NEXT: mov z0.d, p0/m, x0 ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: mov z0.d, p1/m, x0 ; CHECK-NEXT: and z0.d, z0.d, #0x1 ; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 ; CHECK-NEXT: ret @@ -462,9 +462,9 @@ define @test_predicate_insert_2xi1_immediate ( @test_predicate_insert_4xi1_immediate ( %val, i1 %elt) { ; CHECK-LABEL: test_predicate_insert_4xi1_immediate: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: mov w8, #2 // =0x2 ; CHECK-NEXT: index z0.s, #0, #1 +; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: cmpeq p2.s, p1/z, z0.s, z1.s ; CHECK-NEXT: mov z0.s, p0/z, #1 // =0x1 @@ -479,9 +479,9 @@ define @test_predicate_insert_4xi1_immediate ( @test_predicate_insert_8xi1_immediate ( %val, i32 %idx) { ; CHECK-LABEL: test_predicate_insert_8xi1_immediate: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.h -; CHECK-NEXT: mov w8, w0 ; CHECK-NEXT: index z0.h, #0, #1 +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p1.h ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: mov w8, #1 // =0x1 ; CHECK-NEXT: cmpeq p2.h, p1/z, z0.h, z1.h @@ -497,9 +497,9 @@ define @test_predicate_insert_8xi1_immediate ( @test_predicate_insert_16xi1_immediate ( %val) { ; CHECK-LABEL: test_predicate_insert_16xi1_immediate: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.b ; CHECK-NEXT: mov w8, #4 // =0x4 ; CHECK-NEXT: index z0.b, #0, #1 +; CHECK-NEXT: ptrue p1.b ; CHECK-NEXT: mov z1.b, w8 ; CHECK-NEXT: mov w8, wzr ; CHECK-NEXT: cmpeq p2.b, p1/z, z0.b, z1.b @@ -516,9 +516,9 @@ define @test_predicate_insert_16xi1_immediate ( @test_predicate_insert_2xi1( %val, i1 %elt, i32 %idx) { ; CHECK-LABEL: test_predicate_insert_2xi1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.d -; CHECK-NEXT: mov w8, w1 ; CHECK-NEXT: index z0.d, #0, #1 +; CHECK-NEXT: mov w8, w1 +; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 ; CHECK-NEXT: cmpeq p2.d, p1/z, z0.d, z1.d @@ -534,9 +534,9 @@ define @test_predicate_insert_2xi1( %val, i1 define @test_predicate_insert_4xi1( %val, i1 %elt, i32 %idx) { ; CHECK-LABEL: test_predicate_insert_4xi1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.s -; CHECK-NEXT: mov w8, w1 ; CHECK-NEXT: index z0.s, #0, #1 +; CHECK-NEXT: mov w8, w1 +; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: cmpeq p2.s, p1/z, z0.s, z1.s ; CHECK-NEXT: mov z0.s, p0/z, #1 // =0x1 @@ -550,9 +550,9 @@ define @test_predicate_insert_4xi1( %val, i1 define @test_predicate_insert_8xi1( %val, i1 %elt, i32 %idx) { ; CHECK-LABEL: test_predicate_insert_8xi1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.h -; CHECK-NEXT: mov w8, w1 ; CHECK-NEXT: index z0.h, #0, #1 +; CHECK-NEXT: mov w8, w1 +; CHECK-NEXT: ptrue p1.h ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: cmpeq p2.h, p1/z, z0.h, z1.h ; CHECK-NEXT: mov z0.h, p0/z, #1 // =0x1 @@ -567,9 +567,9 @@ define @test_predicate_insert_8xi1( %val, i1 define @test_predicate_insert_16xi1( %val, i1 %elt, i32 %idx) { ; CHECK-LABEL: test_predicate_insert_16xi1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.b -; CHECK-NEXT: mov w8, w1 ; CHECK-NEXT: index z0.b, #0, #1 +; CHECK-NEXT: mov w8, w1 +; CHECK-NEXT: ptrue p1.b ; CHECK-NEXT: mov z1.b, w8 ; CHECK-NEXT: cmpeq p2.b, p1/z, z0.b, z1.b ; CHECK-NEXT: mov z0.b, p0/z, #1 // =0x1 @@ -589,24 +589,24 @@ define @test_predicate_insert_32xi1( %val, ; CHECK-NEXT: .cfi_offset w29, -16 ; CHECK-NEXT: addvl sp, sp, #-2 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG -; CHECK-NEXT: ptrue p2.b ; CHECK-NEXT: rdvl x8, #2 ; CHECK-NEXT: mov z0.b, p1/z, #1 // =0x1 ; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1 ; CHECK-NEXT: sub x8, x8, #1 ; CHECK-NEXT: mov w9, w1 +; CHECK-NEXT: ptrue p1.b ; CHECK-NEXT: cmp x9, x8 ; CHECK-NEXT: csel x8, x9, x8, lo ; CHECK-NEXT: mov x9, sp -; CHECK-NEXT: st1b { z0.b }, p2, [sp, #1, mul vl] -; CHECK-NEXT: st1b { z1.b }, p2, [sp] +; CHECK-NEXT: st1b { z0.b }, p1, [sp, #1, mul vl] +; CHECK-NEXT: st1b { z1.b }, p1, [sp] ; CHECK-NEXT: strb w0, [x9, x8] -; CHECK-NEXT: ld1b { z0.b }, p2/z, [sp] -; CHECK-NEXT: ld1b { z1.b }, p2/z, [sp, #1, mul vl] +; CHECK-NEXT: ld1b { z0.b }, p1/z, [sp] +; CHECK-NEXT: ld1b { z1.b }, p1/z, [sp, #1, mul vl] ; CHECK-NEXT: and z0.b, z0.b, #0x1 ; CHECK-NEXT: and z1.b, z1.b, #0x1 -; CHECK-NEXT: cmpne p0.b, p2/z, z0.b, #0 -; CHECK-NEXT: cmpne p1.b, p2/z, z1.b, #0 +; CHECK-NEXT: cmpne p0.b, p1/z, z0.b, #0 +; CHECK-NEXT: cmpne p1.b, p1/z, z1.b, #0 ; CHECK-NEXT: addvl sp, sp, #2 ; CHECK-NEXT: .cfi_def_cfa wsp, 16 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload diff --git a/llvm/test/CodeGen/AArch64/sve-insert-vector.ll b/llvm/test/CodeGen/AArch64/sve-insert-vector.ll index 4a5e272582d8..5efe9e2819d5 100644 --- a/llvm/test/CodeGen/AArch64/sve-insert-vector.ll +++ b/llvm/test/CodeGen/AArch64/sve-insert-vector.ll @@ -17,15 +17,15 @@ define @insert_v2i64_nxv2i64_idx2( %vec, <2 ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: cntd x8 ; CHECK-NEXT: mov w9, #2 // =0x2 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: sub x8, x8, #2 ; CHECK-NEXT: cmp x8, #2 +; CHECK-NEXT: st1d { z0.d }, p0, [sp] ; CHECK-NEXT: csel x8, x8, x9, lo ; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: lsl x8, x8, #3 -; CHECK-NEXT: st1d { z0.d }, p0, [sp] ; CHECK-NEXT: str q1, [x9, x8] ; CHECK-NEXT: ld1d { z0.d }, p0/z, [sp] ; CHECK-NEXT: addvl sp, sp, #1 @@ -51,15 +51,15 @@ define @insert_v4i32_nxv4i32_idx4( %vec, <4 ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: cntw x8 ; CHECK-NEXT: mov w9, #4 // =0x4 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sub x8, x8, #4 ; CHECK-NEXT: cmp x8, #4 +; CHECK-NEXT: st1w { z0.s }, p0, [sp] ; CHECK-NEXT: csel x8, x8, x9, lo ; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: lsl x8, x8, #2 -; CHECK-NEXT: st1w { z0.s }, p0, [sp] ; CHECK-NEXT: str q1, [x9, x8] ; CHECK-NEXT: ld1w { z0.s }, p0/z, [sp] ; CHECK-NEXT: addvl sp, sp, #1 @@ -85,15 +85,15 @@ define @insert_v8i16_nxv8i16_idx8( %vec, <8 ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: cnth x8 ; CHECK-NEXT: mov w9, #8 // =0x8 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: sub x8, x8, #8 ; CHECK-NEXT: cmp x8, #8 +; CHECK-NEXT: st1h { z0.h }, p0, [sp] ; CHECK-NEXT: csel x8, x8, x9, lo ; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: lsl x8, x8, #1 -; CHECK-NEXT: st1h { z0.h }, p0, [sp] ; CHECK-NEXT: str q1, [x9, x8] ; CHECK-NEXT: ld1h { z0.h }, p0/z, [sp] ; CHECK-NEXT: addvl sp, sp, #1 @@ -119,15 +119,15 @@ define @insert_v16i8_nxv16i8_idx16( %vec, < ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: rdvl x8, #1 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov w9, #16 // =0x10 ; CHECK-NEXT: sub x8, x8, #16 -; CHECK-NEXT: mov x10, sp ; CHECK-NEXT: cmp x8, #16 -; CHECK-NEXT: csel x8, x8, x9, lo ; CHECK-NEXT: st1b { z0.b }, p0, [sp] -; CHECK-NEXT: str q1, [x10, x8] +; CHECK-NEXT: csel x8, x8, x9, lo +; CHECK-NEXT: mov x9, sp +; CHECK-NEXT: str q1, [x9, x8] ; CHECK-NEXT: ld1b { z0.b }, p0/z, [sp] ; CHECK-NEXT: addvl sp, sp, #1 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload @@ -239,8 +239,8 @@ define void @insert_v2i64_nxv16i64_lo2(ptr %psv, ptr %out) uwtable { ; CHECK-NEXT: .cfi_offset w29, -16 ; CHECK-NEXT: addvl sp, sp, #-2 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldr q0, [x0] +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: str q0, [sp, #16] ; CHECK-NEXT: ld1d { z0.d }, p0/z, [sp, #1, mul vl] ; CHECK-NEXT: ld1d { z1.d }, p0/z, [sp] diff --git a/llvm/test/CodeGen/AArch64/sve-int-arith-imm.ll b/llvm/test/CodeGen/AArch64/sve-int-arith-imm.ll index c0ddceb42e1d..52bd79e7a7e6 100644 --- a/llvm/test/CodeGen/AArch64/sve-int-arith-imm.ll +++ b/llvm/test/CodeGen/AArch64/sve-int-arith-imm.ll @@ -55,8 +55,8 @@ define @smax_i16_neg( %a) { define @smax_i16_out_of_range( %a) { ; CHECK-LABEL: smax_i16_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: dupm z1.b, #0x1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: smax z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret %elt = insertelement undef, i16 257, i32 0 @@ -93,8 +93,8 @@ define @smax_i32_neg( %a) { define @smax_i32_out_of_range( %a) { ; CHECK-LABEL: smax_i32_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, #-129 // =0xffffffffffffff7f +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: smax z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %elt = insertelement undef, i32 -129, i32 0 @@ -131,8 +131,8 @@ define @smax_i64_neg( %a) { define @smax_i64_out_of_range( %a) { ; CHECK-LABEL: smax_i64_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, #65535 // =0xffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: smax z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret %elt = insertelement undef, i64 65535, i32 0 @@ -196,8 +196,8 @@ define @smin_i16_neg( %a) { define @smin_i16_out_of_range( %a) { ; CHECK-LABEL: smin_i16_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: dupm z1.b, #0x1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: smin z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret %elt = insertelement undef, i16 257, i32 0 @@ -234,8 +234,8 @@ define @smin_i32_neg( %a) { define @smin_i32_out_of_range( %a) { ; CHECK-LABEL: smin_i32_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, #-129 // =0xffffffffffffff7f +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: smin z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %elt = insertelement undef, i32 -129, i32 0 @@ -272,8 +272,8 @@ define @smin_i64_neg( %a) { define @smin_i64_out_of_range( %a) { ; CHECK-LABEL: smin_i64_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, #65535 // =0xffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: smin z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret %elt = insertelement undef, i64 65535, i32 0 @@ -325,8 +325,8 @@ define @umax_i16_pos( %a) { define @umax_i16_out_of_range( %a) { ; CHECK-LABEL: umax_i16_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: dupm z1.b, #0x1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: umax z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret %elt = insertelement undef, i16 257, i32 0 @@ -351,8 +351,8 @@ define @umax_i32_pos( %a) { define @umax_i32_out_of_range( %a) { ; CHECK-LABEL: umax_i32_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #257 // =0x101 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: umax z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret @@ -378,8 +378,8 @@ define @umax_i64_pos( %a) { define @umax_i64_out_of_range( %a) { ; CHECK-LABEL: umax_i64_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, #65535 // =0xffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: umax z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret %elt = insertelement undef, i64 65535, i32 0 @@ -431,8 +431,8 @@ define @umin_i16_pos( %a) { define @umin_i16_out_of_range( %a) { ; CHECK-LABEL: umin_i16_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: dupm z1.b, #0x1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: umin z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret %elt = insertelement undef, i16 257, i32 0 @@ -457,8 +457,8 @@ define @umin_i32_pos( %a) { define @umin_i32_out_of_range( %a) { ; CHECK-LABEL: umin_i32_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #257 // =0x101 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: umin z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret @@ -484,8 +484,8 @@ define @umin_i64_pos( %a) { define @umin_i64_out_of_range( %a) { ; CHECK-LABEL: umin_i64_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, #65535 // =0xffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: umin z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret %elt = insertelement undef, i64 65535, i32 0 @@ -589,8 +589,8 @@ define @mul_i64_pos( %a) { define @mul_i16_range( %a) { ; CHECK-LABEL: mul_i16_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z1.h, #255 // =0xff +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mul z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret %elt = insertelement undef, i16 255, i32 0 @@ -602,8 +602,8 @@ define @mul_i16_range( %a) { define @mul_i32_range( %a) { ; CHECK-LABEL: mul_i32_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, #255 // =0xff +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mul z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %elt = insertelement undef, i32 255, i32 0 @@ -615,8 +615,8 @@ define @mul_i32_range( %a) { define @mul_i64_range( %a) { ; CHECK-LABEL: mul_i64_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, #255 // =0xff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret %elt = insertelement undef, i64 255, i32 0 @@ -766,8 +766,8 @@ define @lsr_i64( %a){ define @sdiv_const( %a) #0 { ; CHECK-LABEL: sdiv_const: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, #3 // =0x3 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret entry: @@ -778,8 +778,8 @@ entry: define @udiv_const( %a) #0 { ; CHECK-LABEL: udiv_const: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, #3 // =0x3 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/AArch64/sve-int-arith.ll b/llvm/test/CodeGen/AArch64/sve-int-arith.ll index fc2672f8c80a..5f92dee3b530 100644 --- a/llvm/test/CodeGen/AArch64/sve-int-arith.ll +++ b/llvm/test/CodeGen/AArch64/sve-int-arith.ll @@ -532,8 +532,8 @@ define @mls_i64( %a, %b, define @muladd_i64_positiveAddend( %a, %b) ; CHECK-LABEL: muladd_i64_positiveAddend: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z2.d, #0xffffffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mad z0.d, p0/m, z1.d, z2.d ; CHECK-NEXT: ret { @@ -545,8 +545,8 @@ define @mls_i64( %a, %b, define @muladd_i64_negativeAddend( %a, %b) ; CHECK-LABEL: muladd_i64_negativeAddend: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z2.d, #0xffffffff00000001 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mad z0.d, p0/m, z1.d, z2.d ; CHECK-NEXT: ret { @@ -559,8 +559,8 @@ define @muladd_i64_negativeAddend( %a, @muladd_i32_positiveAddend( %a, %b) ; CHECK-LABEL: muladd_i32_positiveAddend: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, #0x10000 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mad z0.s, p0/m, z1.s, z2.s ; CHECK-NEXT: ret { @@ -572,8 +572,8 @@ define @muladd_i32_positiveAddend( %a, @muladd_i32_negativeAddend( %a, %b) ; CHECK-LABEL: muladd_i32_negativeAddend: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, #0xffff0000 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mad z0.s, p0/m, z1.s, z2.s ; CHECK-NEXT: ret { @@ -585,8 +585,8 @@ define @muladd_i32_negativeAddend( %a, @muladd_i16_positiveAddend( %a, %b) ; CHECK-LABEL: muladd_i16_positiveAddend: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, #255 // =0xff +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mad z0.h, p0/m, z1.h, z2.h ; CHECK-NEXT: ret { @@ -598,8 +598,8 @@ define @muladd_i16_positiveAddend( %a, @muladd_i16_negativeAddend( %a, %b) ; CHECK-LABEL: muladd_i16_negativeAddend: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, #-255 // =0xffffffffffffff01 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mad z0.h, p0/m, z1.h, z2.h ; CHECK-NEXT: ret { @@ -611,8 +611,8 @@ define @muladd_i16_negativeAddend( %a, @muladd_i8_positiveAddend( %a, %b) ; CHECK-LABEL: muladd_i8_positiveAddend: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z2.b, #15 // =0xf +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mad z0.b, p0/m, z1.b, z2.b ; CHECK-NEXT: ret { @@ -624,8 +624,8 @@ define @muladd_i8_positiveAddend( %a, @muladd_i8_negativeAddend( %a, %b) ; CHECK-LABEL: muladd_i8_negativeAddend: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z2.b, #-15 // =0xfffffffffffffff1 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mad z0.b, p0/m, z1.b, z2.b ; CHECK-NEXT: ret { @@ -748,8 +748,8 @@ define @mulsub_i8_negativeAddend( %a, @multiple_fused_ops( %a, %b) ; CHECK-LABEL: multiple_fused_ops: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w8, #200 // =0xc8 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w8 ; CHECK-NEXT: mla z2.h, p0/m, z0.h, z1.h ; CHECK-NEXT: mul z0.h, p0/m, z0.h, z2.h @@ -770,8 +770,8 @@ define void @mad_in_loop(ptr %dst, ptr %src1, ptr %src2, i32 %n) { ; CHECK-NEXT: b.lt .LBB70_3 ; CHECK-NEXT: // %bb.1: // %for.body.preheader ; CHECK-NEXT: mov w9, w3 -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z0.s, #1 // =0x1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: whilelo p1.s, xzr, x9 ; CHECK-NEXT: mov x8, xzr ; CHECK-NEXT: cntw x10 diff --git a/llvm/test/CodeGen/AArch64/sve-int-reduce.ll b/llvm/test/CodeGen/AArch64/sve-int-reduce.ll index d04da6245177..8c1b5225b7f2 100644 --- a/llvm/test/CodeGen/AArch64/sve-int-reduce.ll +++ b/llvm/test/CodeGen/AArch64/sve-int-reduce.ll @@ -378,29 +378,29 @@ declare i8 @llvm.vector.reduce.smin.nxv10i8() define i8 @smin_nxv10i8( %a) { ; CHECK-LABEL: smin_nxv10i8: ; CHECK: // %bb.0: -; CHECK-NEXT: uunpkhi z2.h, z0.b -; CHECK-NEXT: mov z1.d, #127 // =0x7f +; CHECK-NEXT: uunpkhi z1.h, z0.b +; CHECK-NEXT: mov z3.d, #127 // =0x7f ; CHECK-NEXT: uunpklo z0.h, z0.b ; CHECK-NEXT: ptrue p0.b -; CHECK-NEXT: uunpklo z3.s, z2.h -; CHECK-NEXT: uunpkhi z2.s, z2.h -; CHECK-NEXT: uunpklo z3.d, z3.s -; CHECK-NEXT: uzp1 z3.s, z3.s, z1.s -; CHECK-NEXT: uzp1 z2.h, z3.h, z2.h -; CHECK-NEXT: uzp1 z2.b, z0.b, z2.b -; CHECK-NEXT: uunpkhi z2.h, z2.b -; CHECK-NEXT: uunpkhi z3.s, z2.h -; CHECK-NEXT: uunpklo z2.s, z2.h -; CHECK-NEXT: uunpkhi z3.d, z3.s -; CHECK-NEXT: uzp1 z3.s, z1.s, z3.s -; CHECK-NEXT: uzp1 z2.h, z2.h, z3.h -; CHECK-NEXT: uzp1 z2.b, z0.b, z2.b -; CHECK-NEXT: uunpkhi z2.h, z2.b -; CHECK-NEXT: uunpkhi z3.s, z2.h -; CHECK-NEXT: uunpklo z2.s, z2.h -; CHECK-NEXT: uunpklo z3.d, z3.s -; CHECK-NEXT: uzp1 z1.s, z3.s, z1.s +; CHECK-NEXT: uunpklo z2.s, z1.h +; CHECK-NEXT: uunpkhi z1.s, z1.h +; CHECK-NEXT: uunpklo z2.d, z2.s +; CHECK-NEXT: uzp1 z2.s, z2.s, z3.s ; CHECK-NEXT: uzp1 z1.h, z2.h, z1.h +; CHECK-NEXT: uzp1 z1.b, z0.b, z1.b +; CHECK-NEXT: uunpkhi z1.h, z1.b +; CHECK-NEXT: uunpkhi z2.s, z1.h +; CHECK-NEXT: uunpklo z1.s, z1.h +; CHECK-NEXT: uunpkhi z2.d, z2.s +; CHECK-NEXT: uzp1 z2.s, z3.s, z2.s +; CHECK-NEXT: uzp1 z1.h, z1.h, z2.h +; CHECK-NEXT: uzp1 z1.b, z0.b, z1.b +; CHECK-NEXT: uunpkhi z1.h, z1.b +; CHECK-NEXT: uunpkhi z2.s, z1.h +; CHECK-NEXT: uunpklo z1.s, z1.h +; CHECK-NEXT: uunpklo z2.d, z2.s +; CHECK-NEXT: uzp1 z2.s, z2.s, z3.s +; CHECK-NEXT: uzp1 z1.h, z1.h, z2.h ; CHECK-NEXT: uzp1 z0.b, z0.b, z1.b ; CHECK-NEXT: sminv b0, p0, z0.b ; CHECK-NEXT: fmov w0, s0 @@ -414,12 +414,12 @@ declare i8 @llvm.vector.reduce.add.nxv12i8() define i8 @uaddv_nxv12i8( %a) { ; CHECK-LABEL: uaddv_nxv12i8: ; CHECK: // %bb.0: -; CHECK-NEXT: uunpkhi z2.h, z0.b -; CHECK-NEXT: mov z1.s, #0 // =0x0 +; CHECK-NEXT: uunpkhi z1.h, z0.b +; CHECK-NEXT: mov z2.s, #0 // =0x0 ; CHECK-NEXT: uunpklo z0.h, z0.b ; CHECK-NEXT: ptrue p0.b -; CHECK-NEXT: uunpklo z2.s, z2.h -; CHECK-NEXT: uzp1 z1.h, z2.h, z1.h +; CHECK-NEXT: uunpklo z1.s, z1.h +; CHECK-NEXT: uzp1 z1.h, z1.h, z2.h ; CHECK-NEXT: uzp1 z0.b, z0.b, z1.b ; CHECK-NEXT: uaddv d0, p0, z0.b ; CHECK-NEXT: fmov x0, d0 @@ -434,15 +434,15 @@ declare i8 @llvm.vector.reduce.umax.nxv14i8() define i8 @umax_nxv14i8( %a) { ; CHECK-LABEL: umax_nxv14i8: ; CHECK: // %bb.0: -; CHECK-NEXT: uunpkhi z2.h, z0.b -; CHECK-NEXT: mov z1.d, #0 // =0x0 +; CHECK-NEXT: uunpkhi z1.h, z0.b +; CHECK-NEXT: mov z3.d, #0 // =0x0 ; CHECK-NEXT: uunpklo z0.h, z0.b ; CHECK-NEXT: ptrue p0.b -; CHECK-NEXT: uunpkhi z3.s, z2.h -; CHECK-NEXT: uunpklo z2.s, z2.h -; CHECK-NEXT: uunpklo z3.d, z3.s -; CHECK-NEXT: uzp1 z1.s, z3.s, z1.s -; CHECK-NEXT: uzp1 z1.h, z2.h, z1.h +; CHECK-NEXT: uunpkhi z2.s, z1.h +; CHECK-NEXT: uunpklo z1.s, z1.h +; CHECK-NEXT: uunpklo z2.d, z2.s +; CHECK-NEXT: uzp1 z2.s, z2.s, z3.s +; CHECK-NEXT: uzp1 z1.h, z1.h, z2.h ; CHECK-NEXT: uzp1 z0.b, z0.b, z1.b ; CHECK-NEXT: umaxv b0, p0, z0.b ; CHECK-NEXT: fmov w0, s0 diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-index.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-index.ll index 2464eacd185d..bc94c087ef5f 100644 --- a/llvm/test/CodeGen/AArch64/sve-intrinsics-index.ll +++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-index.ll @@ -239,10 +239,10 @@ define @index_rr_i32_combine(i32 %a, i32 %b) { define @index_rr_i32_not_combine(i32 %a, i32 %b) { ; CHECK-LABEL: index_rr_i32_not_combine: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: index z0.s, #0, #1 ; CHECK-NEXT: mov z1.s, w0 ; CHECK-NEXT: mov z2.s, w1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mla z1.s, p0/m, z0.s, z2.s ; CHECK-NEXT: add z0.s, z1.s, z0.s ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-imm.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-imm.ll index 3e453a6b7817..5648e8244e6e 100644 --- a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-imm.ll +++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-imm.ll @@ -247,8 +247,8 @@ define @sub_i32_ptrue_all_h( %a) #0 { define @sub_i32_ptrue_all_d( %a) #0 { ; CHECK-LABEL: sub_i32_ptrue_all_d: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.s, #1 // =0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: sub z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %pg.d = tail call @llvm.aarch64.sve.ptrue.nxv2i1(i32 31) @@ -402,8 +402,8 @@ define @subr_i32_ptrue_all_h( %a) #0 { define @subr_i32_ptrue_all_d( %a) #0 { ; CHECK-LABEL: subr_i32_ptrue_all_d: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.s, #1 // =0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: subr z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %pg.d = tail call @llvm.aarch64.sve.ptrue.nxv2i1(i32 31) @@ -449,8 +449,8 @@ define @smax_i16( %a) { define @smax_i16_out_of_range( %a) { ; CHECK-LABEL: smax_i16_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w8, #129 // =0x81 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: smax z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret @@ -480,8 +480,8 @@ define @smax_i32( %a) { define @smax_i32_out_of_range( %a) { ; CHECK-LABEL: smax_i32_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, #-129 // =0xffffffffffffff7f +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: smax z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %pg = call @llvm.aarch64.sve.ptrue.nxv4i1(i32 31) @@ -510,8 +510,8 @@ define @smax_i64( %a) { define @smax_i64_out_of_range( %a) { ; CHECK-LABEL: smax_i64_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, #65535 // =0xffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: smax z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret %pg = call @llvm.aarch64.sve.ptrue.nxv2i1(i32 31) @@ -559,8 +559,8 @@ define @smax_i32_ptrue_all_h( %a) #0 { define @smax_i32_ptrue_all_d( %a) #0 { ; CHECK-LABEL: smax_i32_ptrue_all_d: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.s, #1 // =0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: smax z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %pg.d = tail call @llvm.aarch64.sve.ptrue.nxv2i1(i32 31) @@ -606,8 +606,8 @@ define @smin_i16( %a) { define @smin_i16_out_of_range( %a) { ; CHECK-LABEL: smin_i16_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z1.h, #-129 // =0xffffffffffffff7f +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: smin z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret %pg = call @llvm.aarch64.sve.ptrue.nxv8i1(i32 31) @@ -636,8 +636,8 @@ define @smin_i32( %a) { define @smin_i32_out_of_range( %a) { ; CHECK-LABEL: smin_i32_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #257 // =0x101 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: smin z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret @@ -668,8 +668,8 @@ define @smin_i64( %a) { define @smin_i64_out_of_range( %a) { ; CHECK-LABEL: smin_i64_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, #-256 // =0xffffffffffffff00 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: smin z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret %pg = call @llvm.aarch64.sve.ptrue.nxv2i1(i32 31) @@ -717,8 +717,8 @@ define @smin_i32_ptrue_all_h( %a) #0 { define @smin_i32_ptrue_all_d( %a) #0 { ; CHECK-LABEL: smin_i32_ptrue_all_d: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.s, #1 // =0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: smin z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %pg.d = tail call @llvm.aarch64.sve.ptrue.nxv2i1(i32 31) @@ -764,8 +764,8 @@ define @umax_i16( %a) { define @umax_i16_out_of_range( %a) { ; CHECK-LABEL: umax_i16_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: dupm z1.b, #0x1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: umax z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret %pg = call @llvm.aarch64.sve.ptrue.nxv8i1(i32 31) @@ -794,8 +794,8 @@ define @umax_i32( %a) { define @umax_i32_out_of_range( %a) { ; CHECK-LABEL: umax_i32_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #257 // =0x101 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: umax z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret @@ -825,8 +825,8 @@ define @umax_i64( %a) { define @umax_i64_out_of_range( %a) { ; CHECK-LABEL: umax_i64_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, #65535 // =0xffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: umax z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret %pg = call @llvm.aarch64.sve.ptrue.nxv2i1(i32 31) @@ -874,8 +874,8 @@ define @umax_i32_ptrue_all_h( %a) #0 { define @umax_i32_ptrue_all_d( %a) #0 { ; CHECK-LABEL: umax_i32_ptrue_all_d: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.s, #1 // =0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: umax z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %pg.d = tail call @llvm.aarch64.sve.ptrue.nxv2i1(i32 31) @@ -921,8 +921,8 @@ define @umin_i16( %a) { define @umin_i16_out_of_range( %a) { ; CHECK-LABEL: umin_i16_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: dupm z1.b, #0x1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: umin z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret %pg = call @llvm.aarch64.sve.ptrue.nxv8i1(i32 31) @@ -951,8 +951,8 @@ define @umin_i32( %a) { define @umin_i32_out_of_range( %a) { ; CHECK-LABEL: umin_i32_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #257 // =0x101 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: umin z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret @@ -982,8 +982,8 @@ define @umin_i64( %a) { define @umin_i64_out_of_range( %a) { ; CHECK-LABEL: umin_i64_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, #65535 // =0xffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: umin z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret %pg = call @llvm.aarch64.sve.ptrue.nxv2i1(i32 31) @@ -1031,8 +1031,8 @@ define @umin_i32_ptrue_all_h( %a) #0 { define @umin_i32_ptrue_all_d( %a) #0 { ; CHECK-LABEL: umin_i32_ptrue_all_d: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.s, #1 // =0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: umin z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %pg.d = tail call @llvm.aarch64.sve.ptrue.nxv2i1(i32 31) @@ -2120,8 +2120,8 @@ define @mul_i32_ptrue_all_h( %a) #0 { define @mul_i32_ptrue_all_d( %a) #0 { ; CHECK-LABEL: mul_i32_ptrue_all_d: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.s, #1 // =0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mul z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %pg.d = tail call @llvm.aarch64.sve.ptrue.nxv2i1(i32 31) diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-logical-imm.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-logical-imm.ll index 7cdedeee2cad..5db7ee75c2a8 100644 --- a/llvm/test/CodeGen/AArch64/sve-intrinsics-logical-imm.ll +++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-logical-imm.ll @@ -261,8 +261,8 @@ define @orr_i32_ptrue_all_h( %a) { define @orr_i32_ptrue_all_d( %a) { ; CHECK-LABEL: orr_i32_ptrue_all_d: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.s, #65535 // =0xffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: orr z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %pg.d = tail call @llvm.aarch64.sve.ptrue.nxv2i1(i32 31) diff --git a/llvm/test/CodeGen/AArch64/sve-ld-post-inc.ll b/llvm/test/CodeGen/AArch64/sve-ld-post-inc.ll index da5dc5c5b34d..619134dc4a69 100644 --- a/llvm/test/CodeGen/AArch64/sve-ld-post-inc.ll +++ b/llvm/test/CodeGen/AArch64/sve-ld-post-inc.ll @@ -38,18 +38,18 @@ define @test_post_ld1_dup(ptr %a, ptr %ptr, i64 %inc) { define void @test_post_ld1_int_fixed(ptr %data, i64 %idx, ptr %addr, ptr %res_ptr) #1 { ; CHECK-LABEL: test_post_ld1_int_fixed: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, #2 // =0x2 ; CHECK-NEXT: index z0.d, #0, #1 -; CHECK-NEXT: ptrue p1.d, vl1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: ldr x8, [x0] -; CHECK-NEXT: ldr x9, [x0, x1, lsl #3] +; CHECK-NEXT: ptrue p2.d, vl1 ; CHECK-NEXT: ld1d { z2.d }, p0/z, [x2] -; CHECK-NEXT: cmpeq p2.d, p0/z, z0.d, z1.d +; CHECK-NEXT: ldr x9, [x0, x1, lsl #3] +; CHECK-NEXT: cmpeq p1.d, p0/z, z0.d, z1.d ; CHECK-NEXT: mov z0.d, z2.d -; CHECK-NEXT: mov z2.d, p2/m, x9 -; CHECK-NEXT: mov z0.d, p1/m, x8 +; CHECK-NEXT: mov z0.d, p2/m, x8 +; CHECK-NEXT: mov z2.d, p1/m, x9 ; CHECK-NEXT: add z0.d, z0.d, z2.d ; CHECK-NEXT: st1d { z0.d }, p0, [x3] ; CHECK-NEXT: ret @@ -67,18 +67,18 @@ define void @test_post_ld1_int_fixed(ptr %data, i64 %idx, ptr %addr, ptr %res_pt define void @test_post_ld1_double_fixed(ptr %data, i64 %idx, ptr %addr, ptr %res_ptr) #1 { ; CHECK-LABEL: test_post_ld1_double_fixed: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, #2 // =0x2 ; CHECK-NEXT: index z0.d, #0, #1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 -; CHECK-NEXT: ptrue p1.d, vl1 -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x2] -; CHECK-NEXT: cmpeq p2.d, p0/z, z0.d, z1.d -; CHECK-NEXT: ldr d0, [x0] -; CHECK-NEXT: ldr d1, [x0, x1, lsl #3] -; CHECK-NEXT: sel z0.d, p1, z0.d, z2.d -; CHECK-NEXT: mov z2.d, p2/m, d1 -; CHECK-NEXT: fadd z0.d, z0.d, z2.d +; CHECK-NEXT: ptrue p2.d, vl1 +; CHECK-NEXT: ldr d2, [x0, x1, lsl #3] +; CHECK-NEXT: cmpeq p1.d, p0/z, z0.d, z1.d +; CHECK-NEXT: ld1d { z0.d }, p0/z, [x2] +; CHECK-NEXT: ldr d1, [x0] +; CHECK-NEXT: sel z1.d, p2, z1.d, z0.d +; CHECK-NEXT: mov z0.d, p1/m, d2 +; CHECK-NEXT: fadd z0.d, z1.d, z0.d ; CHECK-NEXT: st1d { z0.d }, p0, [x3] ; CHECK-NEXT: ret %A = load <4 x double>, ptr %addr diff --git a/llvm/test/CodeGen/AArch64/sve-ld1r.ll b/llvm/test/CodeGen/AArch64/sve-ld1r.ll index e42e2272a2d4..fbe82e8591fd 100644 --- a/llvm/test/CodeGen/AArch64/sve-ld1r.ll +++ b/llvm/test/CodeGen/AArch64/sve-ld1r.ll @@ -20,8 +20,8 @@ define @ld1r_stack() { ; CHECK: // %bb.0: ; CHECK-NEXT: sub sp, sp, #16 ; CHECK-NEXT: .cfi_def_cfa_offset 16 -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: adrp x8, :got:g8 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: ldr x8, [x8, :got_lo12:g8] ; CHECK-NEXT: ldrb w8, [x8] ; CHECK-NEXT: strb w8, [sp, #12] @@ -1433,10 +1433,10 @@ define ptr @avoid_preindex_load(ptr %src, ptr %out) { define ptr @avoid_preindex_load_dup(ptr %src, %pg, ptr %out) { ; CHECK-LABEL: avoid_preindex_load_dup: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: ld1rsb { z0.d }, p0/z, [x0, #1] +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: add x0, x0, #1 -; CHECK-NEXT: st1d { z0.d }, p1, [x1] +; CHECK-NEXT: st1d { z0.d }, p0, [x1] ; CHECK-NEXT: ret %ptr = getelementptr inbounds i8, ptr %src, i64 1 %tmp = load i8, ptr %ptr, align 4 @@ -1450,10 +1450,10 @@ define ptr @avoid_preindex_load_dup(ptr %src, %pg, ptr %out) { define ptr @avoid_preindex_load_dup_passthru_zero(ptr %src, %pg, ptr %out) { ; CHECK-LABEL: avoid_preindex_load_dup_passthru_zero: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: ld1rsb { z0.d }, p0/z, [x0, #1] +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: add x0, x0, #1 -; CHECK-NEXT: st1d { z0.d }, p1, [x1] +; CHECK-NEXT: st1d { z0.d }, p0, [x1] ; CHECK-NEXT: ret %ptr = getelementptr inbounds i8, ptr %src, i64 1 %tmp = load i8, ptr %ptr, align 4 @@ -1467,10 +1467,10 @@ define ptr @avoid_preindex_load_dup_passthru_zero(ptr %src, %p define ptr @preindex_load_dup_passthru( %passthru, ptr %src, %pg, ptr %out) { ; CHECK-LABEL: preindex_load_dup_passthru: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: ldrsb x8, [x0, #1]! ; CHECK-NEXT: mov z0.d, p0/m, x8 -; CHECK-NEXT: st1d { z0.d }, p1, [x1] +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: st1d { z0.d }, p0, [x1] ; CHECK-NEXT: ret %ptr = getelementptr inbounds i8, ptr %src, i64 1 %tmp = load i8, ptr %ptr, align 4 @@ -1485,8 +1485,8 @@ define ptr @preindex_load_dup_passthru( %passthru, ptr %src, < define ptr @preidx8sext64_instead_of_ld1r(ptr %src, ptr %out, ptr %dst) { ; CHECK-LABEL: preidx8sext64_instead_of_ld1r: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldrsb x8, [x0, #1]! +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z0.d, x8 ; CHECK-NEXT: st1d { z0.d }, p0, [x1] ; CHECK-NEXT: str x8, [x2] diff --git a/llvm/test/CodeGen/AArch64/sve-lsr-scaled-index-addressing-mode.ll b/llvm/test/CodeGen/AArch64/sve-lsr-scaled-index-addressing-mode.ll index 06ec13280815..5d53c00c5272 100644 --- a/llvm/test/CodeGen/AArch64/sve-lsr-scaled-index-addressing-mode.ll +++ b/llvm/test/CodeGen/AArch64/sve-lsr-scaled-index-addressing-mode.ll @@ -38,8 +38,8 @@ define void @ld_st_nxv8i16(ptr %in, ptr %out) { ; ; ASM-LABEL: ld_st_nxv8i16: ; ASM: // %bb.0: // %entry -; ASM-NEXT: ptrue p0.h ; ASM-NEXT: mov z0.h, #3 // =0x3 +; ASM-NEXT: ptrue p0.h ; ASM-NEXT: mov x8, xzr ; ASM-NEXT: cnth x9 ; ASM-NEXT: .LBB0_1: // %loop @@ -111,8 +111,8 @@ define void @masked_ld_st_nxv8i16(ptr %in, ptr %out, i64 %n) { ; ; ASM-LABEL: masked_ld_st_nxv8i16: ; ASM: // %bb.0: // %entry -; ASM-NEXT: ptrue p0.h ; ASM-NEXT: mov z0.h, #3 // =0x3 +; ASM-NEXT: ptrue p0.h ; ASM-NEXT: mov x8, xzr ; ASM-NEXT: cnth x9 ; ASM-NEXT: .LBB1_1: // %loop diff --git a/llvm/test/CodeGen/AArch64/sve-masked-gather-legalize.ll b/llvm/test/CodeGen/AArch64/sve-masked-gather-legalize.ll index e40d65efb158..dfdfc456ccdb 100644 --- a/llvm/test/CodeGen/AArch64/sve-masked-gather-legalize.ll +++ b/llvm/test/CodeGen/AArch64/sve-masked-gather-legalize.ll @@ -126,9 +126,9 @@ define @masked_gather_nxv8f16( %ptrs, @masked_gather_nxv8bf16(ptr %base, %indices, %mask) #0 { ; CHECK-LABEL: masked_gather_nxv8bf16: ; CHECK: // %bb.0: -; CHECK-NEXT: punpkhi p1.h, p0.b ; CHECK-NEXT: sunpkhi z1.s, z0.h ; CHECK-NEXT: sunpklo z0.s, z0.h +; CHECK-NEXT: punpkhi p1.h, p0.b ; CHECK-NEXT: punpklo p0.h, p0.b ; CHECK-NEXT: ld1h { z1.s }, p1/z, [x0, z1.s, sxtw #1] ; CHECK-NEXT: ld1h { z0.s }, p0/z, [x0, z0.s, sxtw #1] @@ -175,14 +175,14 @@ define @masked_gather_nxv8f32(ptr %base, define @masked_gather_nxv16i8(ptr %base, %indices, %mask) #0 { ; CHECK-LABEL: masked_gather_nxv16i8: ; CHECK: // %bb.0: -; CHECK-NEXT: punpkhi p1.h, p0.b ; CHECK-NEXT: sunpkhi z1.h, z0.b +; CHECK-NEXT: punpkhi p1.h, p0.b ; CHECK-NEXT: sunpklo z0.h, z0.b ; CHECK-NEXT: punpklo p0.h, p0.b -; CHECK-NEXT: sunpkhi z2.s, z1.h -; CHECK-NEXT: sunpklo z1.s, z1.h ; CHECK-NEXT: punpkhi p2.h, p1.b ; CHECK-NEXT: punpklo p1.h, p1.b +; CHECK-NEXT: sunpkhi z2.s, z1.h +; CHECK-NEXT: sunpklo z1.s, z1.h ; CHECK-NEXT: ld1b { z2.s }, p2/z, [x0, z2.s, sxtw] ; CHECK-NEXT: ld1b { z1.s }, p1/z, [x0, z1.s, sxtw] ; CHECK-NEXT: punpkhi p1.h, p0.b diff --git a/llvm/test/CodeGen/AArch64/sve-masked-ldst-sext.ll b/llvm/test/CodeGen/AArch64/sve-masked-ldst-sext.ll index 40d889f1b501..d397424cb162 100644 --- a/llvm/test/CodeGen/AArch64/sve-masked-ldst-sext.ll +++ b/llvm/test/CodeGen/AArch64/sve-masked-ldst-sext.ll @@ -235,9 +235,9 @@ define @masked_sload_x2_8i8_8i64(ptr %a, ptr %b, @masked_zload_nxv8i16(ptr %a, %mask) define @masked_zload_2i16_2f64(ptr noalias %in, %mask) { ; CHECK-LABEL: masked_zload_2i16_2f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0] -; CHECK-NEXT: ucvtf z0.d, p1/m, z0.d +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: ucvtf z0.d, p0/m, z0.d ; CHECK-NEXT: ret %wide.load = call @llvm.masked.load.nxv2i16(ptr %in, i32 2, %mask, undef) %zext = zext %wide.load to @@ -230,9 +230,9 @@ define @masked_zload_x2_8i8_8i64(ptr %a, ptr %b, %data, ptr %base, %offsets, %mask) #0 { ; CHECK-LABEL: masked_scatter_nxv16i8: ; CHECK: // %bb.0: -; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: sunpklo z2.h, z1.b ; CHECK-NEXT: uunpklo z4.h, z0.b -; CHECK-NEXT: punpkhi p0.h, p0.b +; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: sunpkhi z1.h, z1.b ; CHECK-NEXT: uunpkhi z0.h, z0.b +; CHECK-NEXT: punpkhi p0.h, p0.b +; CHECK-NEXT: punpklo p2.h, p1.b +; CHECK-NEXT: punpkhi p1.h, p1.b ; CHECK-NEXT: sunpklo z3.s, z2.h ; CHECK-NEXT: uunpklo z5.s, z4.h ; CHECK-NEXT: sunpkhi z2.s, z2.h -; CHECK-NEXT: punpklo p2.h, p1.b -; CHECK-NEXT: punpkhi p1.h, p1.b ; CHECK-NEXT: st1b { z5.s }, p2, [x0, z3.s, sxtw] ; CHECK-NEXT: uunpkhi z3.s, z4.h ; CHECK-NEXT: st1b { z3.s }, p1, [x0, z2.s, sxtw] -; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: sunpklo z2.s, z1.h -; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: uunpklo z3.s, z0.h ; CHECK-NEXT: sunpkhi z1.s, z1.h ; CHECK-NEXT: uunpkhi z0.s, z0.h +; CHECK-NEXT: punpklo p1.h, p0.b +; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: st1b { z3.s }, p1, [x0, z2.s, sxtw] ; CHECK-NEXT: st1b { z0.s }, p0, [x0, z1.s, sxtw] ; CHECK-NEXT: ret @@ -40,12 +40,12 @@ define void @masked_scatter_nxv16i8( %data, ptr %base, %data, ptr %base, %offsets, %mask) #0 { ; CHECK-LABEL: masked_scatter_nxv8i16: ; CHECK: // %bb.0: -; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: sunpklo z2.s, z1.h ; CHECK-NEXT: uunpklo z3.s, z0.h -; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: sunpkhi z1.s, z1.h ; CHECK-NEXT: uunpkhi z0.s, z0.h +; CHECK-NEXT: punpklo p1.h, p0.b +; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: st1h { z3.s }, p1, [x0, z2.s, sxtw #1] ; CHECK-NEXT: st1h { z0.s }, p0, [x0, z1.s, sxtw #1] ; CHECK-NEXT: ret @@ -57,12 +57,12 @@ define void @masked_scatter_nxv8i16( %data, ptr %base, %data, ptr %base, %offsets, %mask) #0 { ; CHECK-LABEL: masked_scatter_nxv8bf16: ; CHECK: // %bb.0: -; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: sunpklo z2.s, z1.h ; CHECK-NEXT: uunpklo z3.s, z0.h -; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: sunpkhi z1.s, z1.h ; CHECK-NEXT: uunpkhi z0.s, z0.h +; CHECK-NEXT: punpklo p1.h, p0.b +; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: st1h { z3.s }, p1, [x0, z2.s, sxtw #1] ; CHECK-NEXT: st1h { z0.s }, p0, [x0, z1.s, sxtw #1] ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sve-masked-scatter.ll b/llvm/test/CodeGen/AArch64/sve-masked-scatter.ll index e866474942cd..94e525d22b82 100644 --- a/llvm/test/CodeGen/AArch64/sve-masked-scatter.ll +++ b/llvm/test/CodeGen/AArch64/sve-masked-scatter.ll @@ -76,8 +76,8 @@ define void @masked_scatter_nxv2f64( %data, %pg) { ; CHECK-LABEL: masked_scatter_splat_constant_pointer: ; CHECK: // %bb.0: // %vector.body -; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: mov z0.d, #0 // =0x0 +; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: st1w { z0.d }, p1, [z0.d] ; CHECK-NEXT: st1w { z0.d }, p0, [z0.d] diff --git a/llvm/test/CodeGen/AArch64/sve-pr62151.ll b/llvm/test/CodeGen/AArch64/sve-pr62151.ll index 7cec20fda429..5ed34f14a0b1 100644 --- a/llvm/test/CodeGen/AArch64/sve-pr62151.ll +++ b/llvm/test/CodeGen/AArch64/sve-pr62151.ll @@ -5,8 +5,8 @@ define i32 @build_interpolation(<2 x i32> %0, <2 x i32> %1, <2 x i32> %2) { ; CHECK-LABEL: build_interpolation: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: mul v0.2s, v1.2s, v0.2s +; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: // kill: def $d2 killed $d2 def $z2 ; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z2.s ; CHECK-NEXT: mla v0.2s, v1.2s, v0.s[1] diff --git a/llvm/test/CodeGen/AArch64/sve-pred-arith.ll b/llvm/test/CodeGen/AArch64/sve-pred-arith.ll index 4d46ac5ecbaa..6e08606db953 100644 --- a/llvm/test/CodeGen/AArch64/sve-pred-arith.ll +++ b/llvm/test/CodeGen/AArch64/sve-pred-arith.ll @@ -54,24 +54,24 @@ define aarch64_sve_vector_pcs @add_nxv64i1( ; CHECK-NEXT: .cfi_offset w29, -16 ; CHECK-NEXT: addvl sp, sp, #-1 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG -; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: ptrue p4.b ; CHECK-NEXT: str p8, [sp, #3, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: str p7, [sp, #4, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: str p6, [sp, #5, mul vl] // 2-byte Folded Spill +; CHECK-NEXT: ptrue p6.b ; CHECK-NEXT: str p5, [sp, #6, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: ldr p5, [x0] -; CHECK-NEXT: ldr p6, [x1] +; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill +; CHECK-NEXT: ldr p4, [x0] +; CHECK-NEXT: ldr p5, [x1] ; CHECK-NEXT: ldr p7, [x2] ; CHECK-NEXT: ldr p8, [x3] -; CHECK-NEXT: eor p0.b, p4/z, p0.b, p5.b -; CHECK-NEXT: eor p1.b, p4/z, p1.b, p6.b -; CHECK-NEXT: ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: eor p2.b, p4/z, p2.b, p7.b +; CHECK-NEXT: eor p0.b, p6/z, p0.b, p4.b +; CHECK-NEXT: eor p1.b, p6/z, p1.b, p5.b +; CHECK-NEXT: ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload +; CHECK-NEXT: eor p2.b, p6/z, p2.b, p7.b ; CHECK-NEXT: ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: eor p3.b, p4/z, p3.b, p8.b +; CHECK-NEXT: eor p3.b, p6/z, p3.b, p8.b ; CHECK-NEXT: ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload +; CHECK-NEXT: ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: addvl sp, sp, #1 ; CHECK-NEXT: .cfi_def_cfa wsp, 16 @@ -138,24 +138,24 @@ define aarch64_sve_vector_pcs @sub_nxv64i1( ; CHECK-NEXT: .cfi_offset w29, -16 ; CHECK-NEXT: addvl sp, sp, #-1 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG -; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: ptrue p4.b ; CHECK-NEXT: str p8, [sp, #3, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: str p7, [sp, #4, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: str p6, [sp, #5, mul vl] // 2-byte Folded Spill +; CHECK-NEXT: ptrue p6.b ; CHECK-NEXT: str p5, [sp, #6, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: ldr p5, [x0] -; CHECK-NEXT: ldr p6, [x1] +; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill +; CHECK-NEXT: ldr p4, [x0] +; CHECK-NEXT: ldr p5, [x1] ; CHECK-NEXT: ldr p7, [x2] ; CHECK-NEXT: ldr p8, [x3] -; CHECK-NEXT: eor p0.b, p4/z, p0.b, p5.b -; CHECK-NEXT: eor p1.b, p4/z, p1.b, p6.b -; CHECK-NEXT: ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: eor p2.b, p4/z, p2.b, p7.b +; CHECK-NEXT: eor p0.b, p6/z, p0.b, p4.b +; CHECK-NEXT: eor p1.b, p6/z, p1.b, p5.b +; CHECK-NEXT: ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload +; CHECK-NEXT: eor p2.b, p6/z, p2.b, p7.b ; CHECK-NEXT: ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: eor p3.b, p4/z, p3.b, p8.b +; CHECK-NEXT: eor p3.b, p6/z, p3.b, p8.b ; CHECK-NEXT: ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload +; CHECK-NEXT: ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: addvl sp, sp, #1 ; CHECK-NEXT: .cfi_def_cfa wsp, 16 diff --git a/llvm/test/CodeGen/AArch64/sve-pred-selectop.ll b/llvm/test/CodeGen/AArch64/sve-pred-selectop.ll index 600e9c4805ff..8438e9d88f5d 100644 --- a/llvm/test/CodeGen/AArch64/sve-pred-selectop.ll +++ b/llvm/test/CodeGen/AArch64/sve-pred-selectop.ll @@ -322,10 +322,10 @@ entry: define @ornot_v4i32( %z, %x, %y) { ; CHECK-LABEL: ornot_v4i32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z3.s, #-1 // =0xffffffffffffffff -; CHECK-NEXT: eor z2.d, z2.d, z3.d +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, #0 +; CHECK-NEXT: eor z2.d, z2.d, z3.d ; CHECK-NEXT: orr z1.d, z1.d, z2.d ; CHECK-NEXT: mov z0.s, p0/m, z1.s ; CHECK-NEXT: ret @@ -340,10 +340,10 @@ entry: define @ornot_v8i16( %z, %x, %y) { ; CHECK-LABEL: ornot_v8i16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z3.h, #-1 // =0xffffffffffffffff -; CHECK-NEXT: eor z2.d, z2.d, z3.d +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: cmpeq p0.h, p0/z, z0.h, #0 +; CHECK-NEXT: eor z2.d, z2.d, z3.d ; CHECK-NEXT: orr z1.d, z1.d, z2.d ; CHECK-NEXT: mov z0.h, p0/m, z1.h ; CHECK-NEXT: ret @@ -358,10 +358,10 @@ entry: define @ornot_v16i8( %z, %x, %y) { ; CHECK-LABEL: ornot_v16i8: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z3.b, #-1 // =0xffffffffffffffff -; CHECK-NEXT: eor z2.d, z2.d, z3.d +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: cmpeq p0.b, p0/z, z0.b, #0 +; CHECK-NEXT: eor z2.d, z2.d, z3.d ; CHECK-NEXT: orr z1.d, z1.d, z2.d ; CHECK-NEXT: mov z0.b, p0/m, z1.b ; CHECK-NEXT: ret @@ -904,8 +904,8 @@ define @addqr_v4i32( %z, ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, w0 -; CHECK-NEXT: add z1.s, z1.s, z2.s ; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, #0 +; CHECK-NEXT: add z1.s, z1.s, z2.s ; CHECK-NEXT: mov z0.s, p0/m, z1.s ; CHECK-NEXT: ret entry: @@ -922,8 +922,8 @@ define @addqr_v8i16( %z, ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w0 -; CHECK-NEXT: add z1.h, z1.h, z2.h ; CHECK-NEXT: cmpeq p0.h, p0/z, z0.h, #0 +; CHECK-NEXT: add z1.h, z1.h, z2.h ; CHECK-NEXT: mov z0.h, p0/m, z1.h ; CHECK-NEXT: ret entry: @@ -940,8 +940,8 @@ define @addqr_v16i8( %z, ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z2.b, w0 -; CHECK-NEXT: add z1.b, z1.b, z2.b ; CHECK-NEXT: cmpeq p0.b, p0/z, z0.b, #0 +; CHECK-NEXT: add z1.b, z1.b, z2.b ; CHECK-NEXT: mov z0.b, p0/m, z1.b ; CHECK-NEXT: ret entry: @@ -958,8 +958,8 @@ define @subqr_v4i32( %z, ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, w0 -; CHECK-NEXT: sub z1.s, z1.s, z2.s ; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, #0 +; CHECK-NEXT: sub z1.s, z1.s, z2.s ; CHECK-NEXT: mov z0.s, p0/m, z1.s ; CHECK-NEXT: ret entry: @@ -976,8 +976,8 @@ define @subqr_v8i16( %z, ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w0 -; CHECK-NEXT: sub z1.h, z1.h, z2.h ; CHECK-NEXT: cmpeq p0.h, p0/z, z0.h, #0 +; CHECK-NEXT: sub z1.h, z1.h, z2.h ; CHECK-NEXT: mov z0.h, p0/m, z1.h ; CHECK-NEXT: ret entry: @@ -994,8 +994,8 @@ define @subqr_v16i8( %z, ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z2.b, w0 -; CHECK-NEXT: sub z1.b, z1.b, z2.b ; CHECK-NEXT: cmpeq p0.b, p0/z, z0.b, #0 +; CHECK-NEXT: sub z1.b, z1.b, z2.b ; CHECK-NEXT: mov z0.b, p0/m, z1.b ; CHECK-NEXT: ret entry: @@ -1010,10 +1010,10 @@ entry: define @mulqr_v4i32( %z, %x, i32 %y) { ; CHECK-LABEL: mulqr_v4i32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, w0 -; CHECK-NEXT: mul z1.s, z1.s, z2.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, #0 +; CHECK-NEXT: mul z1.s, z1.s, z2.s ; CHECK-NEXT: mov z0.s, p0/m, z1.s ; CHECK-NEXT: ret entry: @@ -1028,10 +1028,10 @@ entry: define @mulqr_v8i16( %z, %x, i16 %y) { ; CHECK-LABEL: mulqr_v8i16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w0 -; CHECK-NEXT: mul z1.h, z1.h, z2.h +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: cmpeq p0.h, p0/z, z0.h, #0 +; CHECK-NEXT: mul z1.h, z1.h, z2.h ; CHECK-NEXT: mov z0.h, p0/m, z1.h ; CHECK-NEXT: ret entry: @@ -1046,10 +1046,10 @@ entry: define @mulqr_v16i8( %z, %x, i8 %y) { ; CHECK-LABEL: mulqr_v16i8: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z2.b, w0 -; CHECK-NEXT: mul z1.b, z1.b, z2.b +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: cmpeq p0.b, p0/z, z0.b, #0 +; CHECK-NEXT: mul z1.b, z1.b, z2.b ; CHECK-NEXT: mov z0.b, p0/m, z1.b ; CHECK-NEXT: ret entry: @@ -1064,11 +1064,11 @@ entry: define @faddqr_v4f32( %z, %x, float %y) { ; CHECK-LABEL: faddqr_v4f32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: // kill: def $s2 killed $s2 def $z2 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, s2 -; CHECK-NEXT: fadd z1.s, z1.s, z2.s ; CHECK-NEXT: fcmeq p0.s, p0/z, z0.s, #0.0 +; CHECK-NEXT: fadd z1.s, z1.s, z2.s ; CHECK-NEXT: mov z0.s, p0/m, z1.s ; CHECK-NEXT: ret entry: @@ -1083,11 +1083,11 @@ entry: define @faddqr_v8f16( %z, %x, half %y) { ; CHECK-LABEL: faddqr_v8f16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: // kill: def $h2 killed $h2 def $z2 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, h2 -; CHECK-NEXT: fadd z1.h, z1.h, z2.h ; CHECK-NEXT: fcmeq p0.h, p0/z, z0.h, #0.0 +; CHECK-NEXT: fadd z1.h, z1.h, z2.h ; CHECK-NEXT: mov z0.h, p0/m, z1.h ; CHECK-NEXT: ret entry: @@ -1102,11 +1102,11 @@ entry: define @fsubqr_v4f32( %z, %x, float %y) { ; CHECK-LABEL: fsubqr_v4f32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: // kill: def $s2 killed $s2 def $z2 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, s2 -; CHECK-NEXT: fsub z1.s, z1.s, z2.s ; CHECK-NEXT: fcmeq p0.s, p0/z, z0.s, #0.0 +; CHECK-NEXT: fsub z1.s, z1.s, z2.s ; CHECK-NEXT: mov z0.s, p0/m, z1.s ; CHECK-NEXT: ret entry: @@ -1121,11 +1121,11 @@ entry: define @fsubqr_v8f16( %z, %x, half %y) { ; CHECK-LABEL: fsubqr_v8f16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: // kill: def $h2 killed $h2 def $z2 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, h2 -; CHECK-NEXT: fsub z1.h, z1.h, z2.h ; CHECK-NEXT: fcmeq p0.h, p0/z, z0.h, #0.0 +; CHECK-NEXT: fsub z1.h, z1.h, z2.h ; CHECK-NEXT: mov z0.h, p0/m, z1.h ; CHECK-NEXT: ret entry: @@ -1140,11 +1140,11 @@ entry: define @fmulqr_v4f32( %z, %x, float %y) { ; CHECK-LABEL: fmulqr_v4f32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: // kill: def $s2 killed $s2 def $z2 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, s2 -; CHECK-NEXT: fmul z1.s, z1.s, z2.s ; CHECK-NEXT: fcmeq p0.s, p0/z, z0.s, #0.0 +; CHECK-NEXT: fmul z1.s, z1.s, z2.s ; CHECK-NEXT: mov z0.s, p0/m, z1.s ; CHECK-NEXT: ret entry: @@ -1159,11 +1159,11 @@ entry: define @fmulqr_v8f16( %z, %x, half %y) { ; CHECK-LABEL: fmulqr_v8f16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: // kill: def $h2 killed $h2 def $z2 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, h2 -; CHECK-NEXT: fmul z1.h, z1.h, z2.h ; CHECK-NEXT: fcmeq p0.h, p0/z, z0.h, #0.0 +; CHECK-NEXT: fmul z1.h, z1.h, z2.h ; CHECK-NEXT: mov z0.h, p0/m, z1.h ; CHECK-NEXT: ret entry: @@ -1178,10 +1178,10 @@ entry: define @sadd_satqr_v4i32( %z, %x, i32 %y) { ; CHECK-LABEL: sadd_satqr_v4i32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, w0 -; CHECK-NEXT: sqadd z1.s, z1.s, z2.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, #0 +; CHECK-NEXT: sqadd z1.s, z1.s, z2.s ; CHECK-NEXT: mov z0.s, p0/m, z1.s ; CHECK-NEXT: ret entry: @@ -1196,10 +1196,10 @@ entry: define @sadd_satqr_v8i16( %z, %x, i16 %y) { ; CHECK-LABEL: sadd_satqr_v8i16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w0 -; CHECK-NEXT: sqadd z1.h, z1.h, z2.h +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: cmpeq p0.h, p0/z, z0.h, #0 +; CHECK-NEXT: sqadd z1.h, z1.h, z2.h ; CHECK-NEXT: mov z0.h, p0/m, z1.h ; CHECK-NEXT: ret entry: @@ -1214,10 +1214,10 @@ entry: define @sadd_satqr_v16i8( %z, %x, i8 %y) { ; CHECK-LABEL: sadd_satqr_v16i8: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z2.b, w0 -; CHECK-NEXT: sqadd z1.b, z1.b, z2.b +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: cmpeq p0.b, p0/z, z0.b, #0 +; CHECK-NEXT: sqadd z1.b, z1.b, z2.b ; CHECK-NEXT: mov z0.b, p0/m, z1.b ; CHECK-NEXT: ret entry: @@ -1232,10 +1232,10 @@ entry: define @uadd_satqr_v4i32( %z, %x, i32 %y) { ; CHECK-LABEL: uadd_satqr_v4i32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, w0 -; CHECK-NEXT: uqadd z1.s, z1.s, z2.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, #0 +; CHECK-NEXT: uqadd z1.s, z1.s, z2.s ; CHECK-NEXT: mov z0.s, p0/m, z1.s ; CHECK-NEXT: ret entry: @@ -1250,10 +1250,10 @@ entry: define @uadd_satqr_v8i16( %z, %x, i16 %y) { ; CHECK-LABEL: uadd_satqr_v8i16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w0 -; CHECK-NEXT: uqadd z1.h, z1.h, z2.h +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: cmpeq p0.h, p0/z, z0.h, #0 +; CHECK-NEXT: uqadd z1.h, z1.h, z2.h ; CHECK-NEXT: mov z0.h, p0/m, z1.h ; CHECK-NEXT: ret entry: @@ -1268,10 +1268,10 @@ entry: define @uadd_satqr_v16i8( %z, %x, i8 %y) { ; CHECK-LABEL: uadd_satqr_v16i8: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z2.b, w0 -; CHECK-NEXT: uqadd z1.b, z1.b, z2.b +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: cmpeq p0.b, p0/z, z0.b, #0 +; CHECK-NEXT: uqadd z1.b, z1.b, z2.b ; CHECK-NEXT: mov z0.b, p0/m, z1.b ; CHECK-NEXT: ret entry: @@ -1286,10 +1286,10 @@ entry: define @ssub_satqr_v4i32( %z, %x, i32 %y) { ; CHECK-LABEL: ssub_satqr_v4i32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, w0 -; CHECK-NEXT: sqsub z1.s, z1.s, z2.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, #0 +; CHECK-NEXT: sqsub z1.s, z1.s, z2.s ; CHECK-NEXT: mov z0.s, p0/m, z1.s ; CHECK-NEXT: ret entry: @@ -1304,10 +1304,10 @@ entry: define @ssub_satqr_v8i16( %z, %x, i16 %y) { ; CHECK-LABEL: ssub_satqr_v8i16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w0 -; CHECK-NEXT: sqsub z1.h, z1.h, z2.h +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: cmpeq p0.h, p0/z, z0.h, #0 +; CHECK-NEXT: sqsub z1.h, z1.h, z2.h ; CHECK-NEXT: mov z0.h, p0/m, z1.h ; CHECK-NEXT: ret entry: @@ -1322,10 +1322,10 @@ entry: define @ssub_satqr_v16i8( %z, %x, i8 %y) { ; CHECK-LABEL: ssub_satqr_v16i8: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z2.b, w0 -; CHECK-NEXT: sqsub z1.b, z1.b, z2.b +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: cmpeq p0.b, p0/z, z0.b, #0 +; CHECK-NEXT: sqsub z1.b, z1.b, z2.b ; CHECK-NEXT: mov z0.b, p0/m, z1.b ; CHECK-NEXT: ret entry: @@ -1340,10 +1340,10 @@ entry: define @usub_satqr_v4i32( %z, %x, i32 %y) { ; CHECK-LABEL: usub_satqr_v4i32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, w0 -; CHECK-NEXT: uqsub z1.s, z1.s, z2.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, #0 +; CHECK-NEXT: uqsub z1.s, z1.s, z2.s ; CHECK-NEXT: mov z0.s, p0/m, z1.s ; CHECK-NEXT: ret entry: @@ -1358,10 +1358,10 @@ entry: define @usub_satqr_v8i16( %z, %x, i16 %y) { ; CHECK-LABEL: usub_satqr_v8i16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w0 -; CHECK-NEXT: uqsub z1.h, z1.h, z2.h +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: cmpeq p0.h, p0/z, z0.h, #0 +; CHECK-NEXT: uqsub z1.h, z1.h, z2.h ; CHECK-NEXT: mov z0.h, p0/m, z1.h ; CHECK-NEXT: ret entry: @@ -1376,10 +1376,10 @@ entry: define @usub_satqr_v16i8( %z, %x, i8 %y) { ; CHECK-LABEL: usub_satqr_v16i8: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z2.b, w0 -; CHECK-NEXT: uqsub z1.b, z1.b, z2.b +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: cmpeq p0.b, p0/z, z0.b, #0 +; CHECK-NEXT: uqsub z1.b, z1.b, z2.b ; CHECK-NEXT: mov z0.b, p0/m, z1.b ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/AArch64/sve-pred-selectop2.ll b/llvm/test/CodeGen/AArch64/sve-pred-selectop2.ll index 14bc1b45e79e..2541910e080e 100644 --- a/llvm/test/CodeGen/AArch64/sve-pred-selectop2.ll +++ b/llvm/test/CodeGen/AArch64/sve-pred-selectop2.ll @@ -202,9 +202,9 @@ entry: define @sdiv_nxv8i16_x( %x, %y, %n) { ; CHECK-LABEL: sdiv_nxv8i16_x: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sunpkhi z3.s, z1.h ; CHECK-NEXT: sunpkhi z4.s, z0.h +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sunpklo z1.s, z1.h ; CHECK-NEXT: sdivr z3.s, p0/m, z3.s, z4.s ; CHECK-NEXT: sunpklo z4.s, z0.h @@ -288,9 +288,9 @@ entry: define @udiv_nxv8i16_x( %x, %y, %n) { ; CHECK-LABEL: udiv_nxv8i16_x: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uunpkhi z3.s, z1.h ; CHECK-NEXT: uunpkhi z4.s, z0.h +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uunpklo z1.s, z1.h ; CHECK-NEXT: udivr z3.s, p0/m, z3.s, z4.s ; CHECK-NEXT: uunpklo z4.s, z0.h @@ -376,17 +376,17 @@ entry: define @srem_nxv8i16_x( %x, %y, %n) { ; CHECK-LABEL: srem_nxv8i16_x: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sunpkhi z3.s, z1.h ; CHECK-NEXT: sunpkhi z4.s, z0.h +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sunpklo z5.s, z0.h ; CHECK-NEXT: sdivr z3.s, p0/m, z3.s, z4.s ; CHECK-NEXT: sunpklo z4.s, z1.h ; CHECK-NEXT: sdivr z4.s, p0/m, z4.s, z5.s ; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: cmpgt p0.h, p0/z, z2.h, #0 -; CHECK-NEXT: uzp1 z3.h, z4.h, z3.h -; CHECK-NEXT: mls z0.h, p0/m, z3.h, z1.h +; CHECK-NEXT: uzp1 z2.h, z4.h, z3.h +; CHECK-NEXT: mls z0.h, p0/m, z2.h, z1.h ; CHECK-NEXT: ret entry: %c = icmp sgt %n, zeroinitializer @@ -419,8 +419,8 @@ define @srem_nxv16i8_x( %x, %n, zeroinitializer @@ -464,17 +464,17 @@ entry: define @urem_nxv8i16_x( %x, %y, %n) { ; CHECK-LABEL: urem_nxv8i16_x: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uunpkhi z3.s, z1.h ; CHECK-NEXT: uunpkhi z4.s, z0.h +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uunpklo z5.s, z0.h ; CHECK-NEXT: udivr z3.s, p0/m, z3.s, z4.s ; CHECK-NEXT: uunpklo z4.s, z1.h ; CHECK-NEXT: udivr z4.s, p0/m, z4.s, z5.s ; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: cmpgt p0.h, p0/z, z2.h, #0 -; CHECK-NEXT: uzp1 z3.h, z4.h, z3.h -; CHECK-NEXT: mls z0.h, p0/m, z3.h, z1.h +; CHECK-NEXT: uzp1 z2.h, z4.h, z3.h +; CHECK-NEXT: mls z0.h, p0/m, z2.h, z1.h ; CHECK-NEXT: ret entry: %c = icmp sgt %n, zeroinitializer @@ -507,8 +507,8 @@ define @urem_nxv16i8_x( %x, %n, zeroinitializer @@ -1140,8 +1140,8 @@ define @fdiv_nxv8f16_x( %x, @sdiv_nxv8i16_y( %x, %y, %n) { ; CHECK-LABEL: sdiv_nxv8i16_y: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sunpkhi z3.s, z1.h ; CHECK-NEXT: sunpkhi z4.s, z0.h +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sunpklo z0.s, z0.h ; CHECK-NEXT: sdivr z3.s, p0/m, z3.s, z4.s ; CHECK-NEXT: sunpklo z4.s, z1.h @@ -1740,9 +1740,9 @@ entry: define @udiv_nxv8i16_y( %x, %y, %n) { ; CHECK-LABEL: udiv_nxv8i16_y: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uunpkhi z3.s, z1.h ; CHECK-NEXT: uunpkhi z4.s, z0.h +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: udivr z3.s, p0/m, z3.s, z4.s ; CHECK-NEXT: uunpklo z4.s, z1.h @@ -1830,17 +1830,17 @@ entry: define @srem_nxv8i16_y( %x, %y, %n) { ; CHECK-LABEL: srem_nxv8i16_y: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sunpkhi z3.s, z1.h ; CHECK-NEXT: sunpkhi z4.s, z0.h +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sunpklo z5.s, z0.h ; CHECK-NEXT: sdivr z3.s, p0/m, z3.s, z4.s ; CHECK-NEXT: sunpklo z4.s, z1.h ; CHECK-NEXT: sdivr z4.s, p0/m, z4.s, z5.s ; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: cmpgt p0.h, p0/z, z2.h, #0 -; CHECK-NEXT: uzp1 z3.h, z4.h, z3.h -; CHECK-NEXT: msb z1.h, p0/m, z3.h, z0.h +; CHECK-NEXT: uzp1 z2.h, z4.h, z3.h +; CHECK-NEXT: msb z1.h, p0/m, z2.h, z0.h ; CHECK-NEXT: mov z0.d, z1.d ; CHECK-NEXT: ret entry: @@ -1874,8 +1874,8 @@ define @srem_nxv16i8_y( %x, @urem_nxv8i16_y( %x, %y, %n) { ; CHECK-LABEL: urem_nxv8i16_y: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uunpkhi z3.s, z1.h ; CHECK-NEXT: uunpkhi z4.s, z0.h +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uunpklo z5.s, z0.h ; CHECK-NEXT: udivr z3.s, p0/m, z3.s, z4.s ; CHECK-NEXT: uunpklo z4.s, z1.h ; CHECK-NEXT: udivr z4.s, p0/m, z4.s, z5.s ; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: cmpgt p0.h, p0/z, z2.h, #0 -; CHECK-NEXT: uzp1 z3.h, z4.h, z3.h -; CHECK-NEXT: msb z1.h, p0/m, z3.h, z0.h +; CHECK-NEXT: uzp1 z2.h, z4.h, z3.h +; CHECK-NEXT: msb z1.h, p0/m, z2.h, z0.h ; CHECK-NEXT: mov z0.d, z1.d ; CHECK-NEXT: ret entry: @@ -1966,8 +1966,8 @@ define @urem_nxv16i8_y( %x, @fdiv_nxv8f16_y( %x, @fmai_nxv4f32_y( %x, %n, zeroinitializer @@ -2871,8 +2871,8 @@ define @fmai_nxv8f16_y( %x, %n, zeroinitializer @@ -2887,8 +2887,8 @@ define @fmai_nxv2f64_y( %x, %n, zeroinitializer @@ -2903,8 +2903,8 @@ define @fma_nxv4f32_y( %x, %n, zeroinitializer @@ -2920,8 +2920,8 @@ define @fma_nxv8f16_y( %x, %n, zeroinitializer @@ -2937,8 +2937,8 @@ define @fma_nxv2f64_y( %x, %n, zeroinitializer diff --git a/llvm/test/CodeGen/AArch64/sve-pred-selectop3.ll b/llvm/test/CodeGen/AArch64/sve-pred-selectop3.ll index 0f09f7dac298..bafd5abcc7b2 100644 --- a/llvm/test/CodeGen/AArch64/sve-pred-selectop3.ll +++ b/llvm/test/CodeGen/AArch64/sve-pred-selectop3.ll @@ -792,8 +792,8 @@ define @fdiv_nxv8f16_x( %x, @fdiv_nxv8f16_y( %x, @fmai_nxv4f32_y( %x, %n, zeroinitializer @@ -1750,8 +1750,8 @@ define @fmai_nxv8f16_y( %x, %n, zeroinitializer @@ -1766,8 +1766,8 @@ define @fmai_nxv2f64_y( %x, %n, zeroinitializer @@ -1782,8 +1782,8 @@ define @fma_nxv4f32_y( %x, %n, zeroinitializer @@ -1799,8 +1799,8 @@ define @fma_nxv8f16_y( %x, %n, zeroinitializer @@ -1816,8 +1816,8 @@ define @fma_nxv2f64_y( %x, %n, zeroinitializer diff --git a/llvm/test/CodeGen/AArch64/sve-ptest-removal-cmple.ll b/llvm/test/CodeGen/AArch64/sve-ptest-removal-cmple.ll index fbadbf7226fd..8bd38d7bc44d 100644 --- a/llvm/test/CodeGen/AArch64/sve-ptest-removal-cmple.ll +++ b/llvm/test/CodeGen/AArch64/sve-ptest-removal-cmple.ll @@ -320,8 +320,8 @@ define i1 @cmp32_ptest_any_xx( %pg, %a, %pg, %a, %b) { ; CHECK-LABEL: cmp8_ptest_first_ax: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.b ; CHECK-NEXT: cmpge p0.b, p0/z, z0.b, z1.b +; CHECK-NEXT: ptrue p1.b ; CHECK-NEXT: ptest p1, p0.b ; CHECK-NEXT: cset w0, mi ; CHECK-NEXT: ret @@ -338,8 +338,8 @@ define i1 @cmp8_ptest_first_ax( %pg, %a, %pg, %a, %b) { ; CHECK-LABEL: cmp8_ptest_last_ax: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.b ; CHECK-NEXT: cmpge p0.b, p0/z, z0.b, z1.b +; CHECK-NEXT: ptrue p1.b ; CHECK-NEXT: ptest p1, p0.b ; CHECK-NEXT: cset w0, lo ; CHECK-NEXT: ret @@ -371,8 +371,8 @@ define i1 @cmp8_ptest_any_ax( %pg, %a, %pg, %a, %b) { ; CHECK-LABEL: cmp32_ptest_first_ax: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: cmpge p0.s, p0/z, z0.s, z1.s +; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: ptest p1, p0.b ; CHECK-NEXT: cset w0, mi ; CHECK-NEXT: ret @@ -390,8 +390,8 @@ define i1 @cmp32_ptest_first_ax( %pg, %a, < define i1 @cmp32_ptest_last_ax( %pg, %a, %b) { ; CHECK-LABEL: cmp32_ptest_last_ax: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: cmpge p0.s, p0/z, z0.s, z1.s +; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: ptest p1, p0.b ; CHECK-NEXT: cset w0, lo ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sve-redundant-store.ll b/llvm/test/CodeGen/AArch64/sve-redundant-store.ll index 6873404724f1..508fe5d5a58a 100644 --- a/llvm/test/CodeGen/AArch64/sve-redundant-store.ll +++ b/llvm/test/CodeGen/AArch64/sve-redundant-store.ll @@ -35,8 +35,8 @@ entry: define void @keep_scalable_store(ptr writeonly %ptr, ptr %a, %b) { ; CHECK-LABEL: keep_scalable_store: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldp q2, q1, [x1] +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: st1w { z0.s }, p0, [x0] ; CHECK-NEXT: stp q2, q1, [x0] ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sve-split-extract-elt.ll b/llvm/test/CodeGen/AArch64/sve-split-extract-elt.ll index a1c2ec9c7e1d..76190eba870d 100644 --- a/llvm/test/CodeGen/AArch64/sve-split-extract-elt.ll +++ b/llvm/test/CodeGen/AArch64/sve-split-extract-elt.ll @@ -22,15 +22,15 @@ define i8 @split_extract_32i8_idx( %a, i32 %idx) { ; CHECK-NEXT: addvl sp, sp, #-2 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: rdvl x8, #2 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov w9, w0 ; CHECK-NEXT: sub x8, x8, #1 ; CHECK-NEXT: cmp x9, x8 -; CHECK-NEXT: csel x8, x9, x8, lo -; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: st1b { z1.b }, p0, [sp, #1, mul vl] ; CHECK-NEXT: st1b { z0.b }, p0, [sp] +; CHECK-NEXT: csel x8, x9, x8, lo +; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: ldrb w0, [x9, x8] ; CHECK-NEXT: addvl sp, sp, #2 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload @@ -46,15 +46,15 @@ define i16 @split_extract_16i16_idx( %a, i32 %idx) { ; CHECK-NEXT: addvl sp, sp, #-2 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: rdvl x8, #1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w9, w0 ; CHECK-NEXT: sub x8, x8, #1 ; CHECK-NEXT: cmp x9, x8 -; CHECK-NEXT: csel x8, x9, x8, lo -; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: st1h { z1.h }, p0, [sp, #1, mul vl] ; CHECK-NEXT: st1h { z0.h }, p0, [sp] +; CHECK-NEXT: csel x8, x9, x8, lo +; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: ldrh w0, [x9, x8, lsl #1] ; CHECK-NEXT: addvl sp, sp, #2 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload @@ -70,15 +70,15 @@ define i32 @split_extract_8i32_idx( %a, i32 %idx) { ; CHECK-NEXT: addvl sp, sp, #-2 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: cnth x8 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w9, w0 ; CHECK-NEXT: sub x8, x8, #1 ; CHECK-NEXT: cmp x9, x8 -; CHECK-NEXT: csel x8, x9, x8, lo -; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: st1w { z1.s }, p0, [sp, #1, mul vl] ; CHECK-NEXT: st1w { z0.s }, p0, [sp] +; CHECK-NEXT: csel x8, x9, x8, lo +; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: ldr w0, [x9, x8, lsl #2] ; CHECK-NEXT: addvl sp, sp, #2 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload @@ -94,15 +94,15 @@ define i64 @split_extract_8i64_idx( %a, i32 %idx) { ; CHECK-NEXT: addvl sp, sp, #-4 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 32 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: cnth x8 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w9, w0 ; CHECK-NEXT: sub x8, x8, #1 ; CHECK-NEXT: cmp x9, x8 -; CHECK-NEXT: csel x8, x9, x8, lo -; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: st1d { z3.d }, p0, [sp, #3, mul vl] ; CHECK-NEXT: st1d { z2.d }, p0, [sp, #2, mul vl] +; CHECK-NEXT: csel x8, x9, x8, lo +; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: st1d { z1.d }, p0, [sp, #1, mul vl] ; CHECK-NEXT: st1d { z0.d }, p0, [sp] ; CHECK-NEXT: ldr x0, [x9, x8, lsl #3] @@ -140,15 +140,15 @@ define i16 @split_extract_16i16( %a) { ; CHECK-NEXT: addvl sp, sp, #-2 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: rdvl x8, #1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w9, #128 // =0x80 ; CHECK-NEXT: sub x8, x8, #1 ; CHECK-NEXT: cmp x8, #128 -; CHECK-NEXT: csel x8, x8, x9, lo -; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: st1h { z1.h }, p0, [sp, #1, mul vl] ; CHECK-NEXT: st1h { z0.h }, p0, [sp] +; CHECK-NEXT: csel x8, x8, x9, lo +; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: ldrh w0, [x9, x8, lsl #1] ; CHECK-NEXT: addvl sp, sp, #2 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload @@ -164,16 +164,16 @@ define i32 @split_extract_16i32( %a) { ; CHECK-NEXT: addvl sp, sp, #-4 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 32 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: rdvl x8, #1 ; CHECK-NEXT: mov w9, #34464 // =0x86a0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: movk w9, #1, lsl #16 ; CHECK-NEXT: sub x8, x8, #1 ; CHECK-NEXT: cmp x8, x9 -; CHECK-NEXT: csel x8, x8, x9, lo -; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: st1w { z3.s }, p0, [sp, #3, mul vl] ; CHECK-NEXT: st1w { z2.s }, p0, [sp, #2, mul vl] +; CHECK-NEXT: csel x8, x8, x9, lo +; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: st1w { z1.s }, p0, [sp, #1, mul vl] ; CHECK-NEXT: st1w { z0.s }, p0, [sp] ; CHECK-NEXT: ldr w0, [x9, x8, lsl #2] @@ -191,15 +191,15 @@ define i64 @split_extract_4i64( %a) { ; CHECK-NEXT: addvl sp, sp, #-2 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: cntw x8 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w9, #10 // =0xa ; CHECK-NEXT: sub x8, x8, #1 ; CHECK-NEXT: cmp x8, #10 -; CHECK-NEXT: csel x8, x8, x9, lo -; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: st1d { z1.d }, p0, [sp, #1, mul vl] ; CHECK-NEXT: st1d { z0.d }, p0, [sp] +; CHECK-NEXT: csel x8, x8, x9, lo +; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: ldr x0, [x9, x8, lsl #3] ; CHECK-NEXT: addvl sp, sp, #2 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload diff --git a/llvm/test/CodeGen/AArch64/sve-split-fcvt.ll b/llvm/test/CodeGen/AArch64/sve-split-fcvt.ll index 3997409172d0..bc015116917d 100644 --- a/llvm/test/CodeGen/AArch64/sve-split-fcvt.ll +++ b/llvm/test/CodeGen/AArch64/sve-split-fcvt.ll @@ -6,9 +6,9 @@ define @fcvts_nxv8f16( %a) { ; CHECK-LABEL: fcvts_nxv8f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uunpklo z1.s, z0.h ; CHECK-NEXT: uunpkhi z2.s, z0.h +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: movprfx z0, z1 ; CHECK-NEXT: fcvt z0.s, p0/m, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -21,9 +21,9 @@ define @fcvts_nxv8f16( %a) { define @fcvtd_nxv4f16( %a) { ; CHECK-LABEL: fcvtd_nxv4f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: uunpklo z1.d, z0.s ; CHECK-NEXT: uunpkhi z2.d, z0.s +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: movprfx z0, z1 ; CHECK-NEXT: fcvt z0.d, p0/m, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -37,8 +37,8 @@ define @fcvtd_nxv8f16( %a) { ; CHECK-LABEL: fcvtd_nxv8f16: ; CHECK: // %bb.0: ; CHECK-NEXT: uunpklo z1.s, z0.h -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: uunpkhi z0.s, z0.h +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: uunpklo z2.d, z1.s ; CHECK-NEXT: uunpkhi z1.d, z1.s ; CHECK-NEXT: uunpklo z3.d, z0.s @@ -58,9 +58,9 @@ define @fcvtd_nxv8f16( %a) { define @fcvtd_nxv4f32( %a) { ; CHECK-LABEL: fcvtd_nxv4f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: uunpklo z1.d, z0.s ; CHECK-NEXT: uunpkhi z2.d, z0.s +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: movprfx z0, z1 ; CHECK-NEXT: fcvt z0.d, p0/m, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -73,11 +73,11 @@ define @fcvtd_nxv4f32( %a) { define @fcvtd_nxv8f32( %a) { ; CHECK-LABEL: fcvtd_nxv8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: uunpklo z2.d, z0.s ; CHECK-NEXT: uunpkhi z3.d, z0.s ; CHECK-NEXT: uunpklo z4.d, z1.s ; CHECK-NEXT: uunpkhi z5.d, z1.s +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: movprfx z0, z2 ; CHECK-NEXT: fcvt z0.d, p0/m, z2.s ; CHECK-NEXT: movprfx z1, z3 @@ -195,9 +195,9 @@ define @fcvtzs_h_nxv8f64( %a) { define @fcvtzs_d_nxv4f32( %a) { ; CHECK-LABEL: fcvtzs_d_nxv4f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: uunpklo z1.d, z0.s ; CHECK-NEXT: uunpkhi z2.d, z0.s +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: movprfx z0, z1 ; CHECK-NEXT: fcvtzs z0.d, p0/m, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -210,11 +210,11 @@ define @fcvtzs_d_nxv4f32( %a) { define @fcvtzs_s_nxv16f16( %a) { ; CHECK-LABEL: fcvtzs_s_nxv16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uunpklo z2.s, z0.h ; CHECK-NEXT: uunpkhi z3.s, z0.h ; CHECK-NEXT: uunpklo z4.s, z1.h ; CHECK-NEXT: uunpkhi z5.s, z1.h +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: movprfx z0, z2 ; CHECK-NEXT: fcvtzs z0.s, p0/m, z2.h ; CHECK-NEXT: movprfx z1, z3 @@ -247,9 +247,9 @@ define @fcvtzu_s_nxv4f64( %a) { define @fcvtzu_d_nxv4f32( %a) { ; CHECK-LABEL: fcvtzu_d_nxv4f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: uunpklo z1.d, z0.s ; CHECK-NEXT: uunpkhi z2.d, z0.s +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: movprfx z0, z1 ; CHECK-NEXT: fcvtzu z0.d, p0/m, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -295,8 +295,8 @@ define @scvtf_s_nxv16i8( %a) { ; CHECK-LABEL: scvtf_s_nxv16i8: ; CHECK: // %bb.0: ; CHECK-NEXT: sunpklo z1.h, z0.b -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sunpkhi z0.h, z0.b +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sunpklo z2.s, z1.h ; CHECK-NEXT: sunpkhi z1.s, z1.h ; CHECK-NEXT: sunpklo z3.s, z0.h @@ -316,9 +316,9 @@ define @scvtf_s_nxv16i8( %a) { define @scvtf_d_nxv4i32( %a) { ; CHECK-LABEL: scvtf_d_nxv4i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: sunpklo z1.d, z0.s ; CHECK-NEXT: sunpkhi z2.d, z0.s +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: movprfx z0, z1 ; CHECK-NEXT: scvtf z0.d, p0/m, z1.d ; CHECK-NEXT: movprfx z1, z2 @@ -333,8 +333,8 @@ define @scvtf_d_nxv4i1( %a) { ; CHECK: // %bb.0: ; CHECK-NEXT: punpklo p2.h, p0.b ; CHECK-NEXT: punpkhi p0.h, p0.b -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: mov z0.d, p2/z, #-1 // =0xffffffffffffffff +; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: mov z1.d, p0/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: scvtf z0.d, p1/m, z0.d ; CHECK-NEXT: scvtf z1.d, p1/m, z1.d @@ -378,9 +378,9 @@ define @ucvtf_h_nxv8i64( %a) { define @ucvtf_d_nxv4i32( %a) { ; CHECK-LABEL: ucvtf_d_nxv4i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: uunpklo z1.d, z0.s ; CHECK-NEXT: uunpkhi z2.d, z0.s +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: movprfx z0, z1 ; CHECK-NEXT: ucvtf z0.d, p0/m, z1.d ; CHECK-NEXT: movprfx z1, z2 @@ -395,8 +395,8 @@ define @ucvtf_d_nxv4i1( %a) { ; CHECK: // %bb.0: ; CHECK-NEXT: punpklo p2.h, p0.b ; CHECK-NEXT: punpkhi p0.h, p0.b -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: mov z0.d, p2/z, #1 // =0x1 +; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: mov z1.d, p0/z, #1 // =0x1 ; CHECK-NEXT: ucvtf z0.d, p1/m, z0.d ; CHECK-NEXT: ucvtf z1.d, p1/m, z1.d diff --git a/llvm/test/CodeGen/AArch64/sve-split-fp-reduce.ll b/llvm/test/CodeGen/AArch64/sve-split-fp-reduce.ll index 7f642882eddb..696b6c34ef04 100644 --- a/llvm/test/CodeGen/AArch64/sve-split-fp-reduce.ll +++ b/llvm/test/CodeGen/AArch64/sve-split-fp-reduce.ll @@ -23,8 +23,8 @@ define double @fadda_nxv8f64(double %init, %a) { define float @faddv_nxv8f32(float %init, %a) { ; CHECK-LABEL: faddv_nxv8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: fadd z1.s, z1.s, z2.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: faddv s1, p0, z1.s ; CHECK-NEXT: fadd s0, s0, s1 ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sve-split-insert-elt.ll b/llvm/test/CodeGen/AArch64/sve-split-insert-elt.ll index 5441659fa5cb..75366384cb75 100644 --- a/llvm/test/CodeGen/AArch64/sve-split-insert-elt.ll +++ b/llvm/test/CodeGen/AArch64/sve-split-insert-elt.ll @@ -6,9 +6,9 @@ define @promote_insert_8i8( %a, i8 %elt, i64 %idx) { ; CHECK-LABEL: promote_insert_8i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: index z1.h, #0, #1 ; CHECK-NEXT: mov z2.h, w1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: cmpeq p0.h, p0/z, z1.h, z2.h ; CHECK-NEXT: mov z0.h, p0/m, w0 ; CHECK-NEXT: ret @@ -23,13 +23,13 @@ define @split_insert_32i8_idx( %a, i8 %elt, ; CHECK-NEXT: addvl sp, sp, #-2 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: rdvl x8, #2 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: sub x8, x8, #1 ; CHECK-NEXT: cmp x1, x8 -; CHECK-NEXT: csel x8, x1, x8, lo ; CHECK-NEXT: st1b { z1.b }, p0, [sp, #1, mul vl] +; CHECK-NEXT: csel x8, x1, x8, lo ; CHECK-NEXT: st1b { z0.b }, p0, [sp] ; CHECK-NEXT: strb w0, [x9, x8] ; CHECK-NEXT: ld1b { z0.b }, p0/z, [sp] @@ -48,13 +48,13 @@ define @split_insert_8f32_idx( %a, floa ; CHECK-NEXT: addvl sp, sp, #-2 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: cnth x8 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: sub x8, x8, #1 ; CHECK-NEXT: cmp x0, x8 -; CHECK-NEXT: csel x8, x0, x8, lo ; CHECK-NEXT: st1w { z1.s }, p0, [sp, #1, mul vl] +; CHECK-NEXT: csel x8, x0, x8, lo ; CHECK-NEXT: st1w { z0.s }, p0, [sp] ; CHECK-NEXT: str s2, [x9, x8, lsl #2] ; CHECK-NEXT: ld1w { z0.s }, p0/z, [sp] @@ -73,13 +73,13 @@ define @split_insert_8i64_idx( %a, i64 %elt ; CHECK-NEXT: addvl sp, sp, #-4 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 32 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: cnth x8 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: sub x8, x8, #1 ; CHECK-NEXT: cmp x1, x8 -; CHECK-NEXT: csel x8, x1, x8, lo ; CHECK-NEXT: st1d { z3.d }, p0, [sp, #3, mul vl] +; CHECK-NEXT: csel x8, x1, x8, lo ; CHECK-NEXT: st1d { z2.d }, p0, [sp, #2, mul vl] ; CHECK-NEXT: st1d { z1.d }, p0, [sp, #1, mul vl] ; CHECK-NEXT: st1d { z0.d }, p0, [sp] @@ -100,9 +100,9 @@ define @split_insert_8i64_idx( %a, i64 %elt define @promote_insert_4i16( %a, i16 %elt) { ; CHECK-LABEL: promote_insert_4i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #5 // =0x5 ; CHECK-NEXT: index z1.s, #0, #1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, w8 ; CHECK-NEXT: cmpeq p0.s, p0/z, z1.s, z2.s ; CHECK-NEXT: mov z0.s, p0/m, w0 @@ -117,9 +117,9 @@ define @promote_insert_4i16( %a, i16 %elt) define @split_insert_32i8( %a, i8 %elt) { ; CHECK-LABEL: split_insert_32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov w8, #3 // =0x3 ; CHECK-NEXT: index z2.b, #0, #1 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z3.b, w8 ; CHECK-NEXT: cmpeq p0.b, p0/z, z2.b, z3.b ; CHECK-NEXT: mov z0.b, p0/m, w0 @@ -135,14 +135,14 @@ define @split_insert_32i16( %a, i16 %elt) ; CHECK-NEXT: addvl sp, sp, #-4 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 32 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: rdvl x8, #2 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w9, #128 // =0x80 ; CHECK-NEXT: sub x8, x8, #1 ; CHECK-NEXT: cmp x8, #128 +; CHECK-NEXT: st1h { z3.h }, p0, [sp, #3, mul vl] ; CHECK-NEXT: csel x8, x8, x9, lo ; CHECK-NEXT: mov x9, sp -; CHECK-NEXT: st1h { z3.h }, p0, [sp, #3, mul vl] ; CHECK-NEXT: st1h { z2.h }, p0, [sp, #2, mul vl] ; CHECK-NEXT: st1h { z1.h }, p0, [sp, #1, mul vl] ; CHECK-NEXT: st1h { z0.h }, p0, [sp] @@ -165,15 +165,15 @@ define @split_insert_8i32( %a, i32 %elt) { ; CHECK-NEXT: addvl sp, sp, #-2 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: cnth x8 ; CHECK-NEXT: mov w9, #16960 // =0x4240 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: movk w9, #15, lsl #16 ; CHECK-NEXT: sub x8, x8, #1 ; CHECK-NEXT: cmp x8, x9 +; CHECK-NEXT: st1w { z1.s }, p0, [sp, #1, mul vl] ; CHECK-NEXT: csel x8, x8, x9, lo ; CHECK-NEXT: mov x9, sp -; CHECK-NEXT: st1w { z1.s }, p0, [sp, #1, mul vl] ; CHECK-NEXT: st1w { z0.s }, p0, [sp] ; CHECK-NEXT: str w0, [x9, x8, lsl #2] ; CHECK-NEXT: ld1w { z0.s }, p0/z, [sp] diff --git a/llvm/test/CodeGen/AArch64/sve-split-int-reduce.ll b/llvm/test/CodeGen/AArch64/sve-split-int-reduce.ll index 42f3a163d14c..dd7b15ef5ee6 100644 --- a/llvm/test/CodeGen/AArch64/sve-split-int-reduce.ll +++ b/llvm/test/CodeGen/AArch64/sve-split-int-reduce.ll @@ -17,8 +17,8 @@ define i8 @andv_nxv8i8( %a) { define i32 @andv_nxv8i32( %a) { ; CHECK-LABEL: andv_nxv8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and z0.d, z0.d, z1.d +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: andv s0, p0, z0.s ; CHECK-NEXT: fmov w0, s0 ; CHECK-NEXT: ret @@ -71,8 +71,8 @@ define i16 @xorv_nxv2i16( %a) { define i32 @xorv_nxv8i32( %a) { ; CHECK-LABEL: xorv_nxv8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: eor z0.d, z0.d, z1.d +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: eorv s0, p0, z0.s ; CHECK-NEXT: fmov w0, s0 ; CHECK-NEXT: ret @@ -97,8 +97,8 @@ define i16 @uaddv_nxv4i16( %a) { define i16 @uaddv_nxv16i16( %a) { ; CHECK-LABEL: uaddv_nxv16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: add z0.h, z0.h, z1.h +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: uaddv d0, p0, z0.h ; CHECK-NEXT: fmov x0, d0 ; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0 @@ -127,8 +127,8 @@ define i32 @uaddv_nxv16i32( %a) { define i32 @umin_nxv2i32( %a) { ; CHECK-LABEL: umin_nxv2i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and z0.d, z0.d, #0xffffffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: uminv d0, p0, z0.d ; CHECK-NEXT: fmov x0, d0 ; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0 diff --git a/llvm/test/CodeGen/AArch64/sve-split-load.ll b/llvm/test/CodeGen/AArch64/sve-split-load.ll index af03059cf0d8..754f0339702d 100644 --- a/llvm/test/CodeGen/AArch64/sve-split-load.ll +++ b/llvm/test/CodeGen/AArch64/sve-split-load.ll @@ -93,8 +93,8 @@ define @masked_load_split_32i16(ptr %a, % ; CHECK-NEXT: punpklo p2.h, p0.b ; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: punpklo p3.h, p1.b -; CHECK-NEXT: punpkhi p1.h, p1.b ; CHECK-NEXT: ld1h { z0.h }, p2/z, [x0] +; CHECK-NEXT: punpkhi p1.h, p1.b ; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0, #1, mul vl] ; CHECK-NEXT: ld1h { z2.h }, p3/z, [x0, #2, mul vl] ; CHECK-NEXT: ld1h { z3.h }, p1/z, [x0, #3, mul vl] @@ -123,8 +123,8 @@ define @masked_load_split_8i64(ptr %a, %pg) ; CHECK-NEXT: punpklo p2.h, p1.b ; CHECK-NEXT: punpkhi p1.h, p1.b ; CHECK-NEXT: punpklo p3.h, p0.b -; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: ld1d { z0.d }, p2/z, [x0] +; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: ld1d { z1.d }, p1/z, [x0, #1, mul vl] ; CHECK-NEXT: ld1d { z2.d }, p3/z, [x0, #2, mul vl] ; CHECK-NEXT: ld1d { z3.d }, p0/z, [x0, #3, mul vl] diff --git a/llvm/test/CodeGen/AArch64/sve-split-store.ll b/llvm/test/CodeGen/AArch64/sve-split-store.ll index 90ec783ea4db..affa9a18ac18 100644 --- a/llvm/test/CodeGen/AArch64/sve-split-store.ll +++ b/llvm/test/CodeGen/AArch64/sve-split-store.ll @@ -81,8 +81,8 @@ define void @masked_store_split_32i16( %data, ptr %a, %data, ptr %a, %data, ptr %a, i32 1, %pg) diff --git a/llvm/test/CodeGen/AArch64/sve-srem-combine-loop.ll b/llvm/test/CodeGen/AArch64/sve-srem-combine-loop.ll index 9c3d4b1e5a81..f556d60d23b8 100644 --- a/llvm/test/CodeGen/AArch64/sve-srem-combine-loop.ll +++ b/llvm/test/CodeGen/AArch64/sve-srem-combine-loop.ll @@ -6,8 +6,8 @@ target triple = "aarch64-unknown-linux-gnu" define @srem_combine_loop( %a) #0 { ; CHECK-LABEL: srem_combine_loop: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.d, z0.d +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, #2 // =0x2 ; CHECK-NEXT: asrd z1.s, p0/m, z1.s, #1 ; CHECK-NEXT: mls z0.s, p0/m, z1.s, z2.s diff --git a/llvm/test/CodeGen/AArch64/sve-st1-addressing-mode-reg-imm.ll b/llvm/test/CodeGen/AArch64/sve-st1-addressing-mode-reg-imm.ll index 728041d3f916..3273e6b384f6 100644 --- a/llvm/test/CodeGen/AArch64/sve-st1-addressing-mode-reg-imm.ll +++ b/llvm/test/CodeGen/AArch64/sve-st1-addressing-mode-reg-imm.ll @@ -105,8 +105,8 @@ define void @st1d_inbound( %data, ptr %a) { define void @store_nxv2f32(ptr %out) { ; CHECK-LABEL: store_nxv2f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: fmov z0.s, #1.00000000 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: st1w { z0.d }, p0, [x0] ; CHECK-NEXT: ret %ins = insertelement undef, float 1.0, i32 0 @@ -118,8 +118,8 @@ define void @store_nxv2f32(ptr %out) { define void @store_nxv4f16(ptr %out) { ; CHECK-LABEL: store_nxv4f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: fmov z0.h, #1.00000000 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: st1h { z0.s }, p0, [x0] ; CHECK-NEXT: ret %ins = insertelement undef, half 1.0, i32 0 @@ -133,9 +133,9 @@ define void @store_nxv4f16(ptr %out) { define void @store_nxv6f32(ptr %out) { ; CHECK-LABEL: store_nxv6f32: ; CHECK: // %bb.0: +; CHECK-NEXT: fmov z0.s, #1.00000000 ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ptrue p1.s -; CHECK-NEXT: fmov z0.s, #1.00000000 ; CHECK-NEXT: st1w { z0.d }, p0, [x0, #2, mul vl] ; CHECK-NEXT: st1w { z0.s }, p1, [x0] ; CHECK-NEXT: ret @@ -148,9 +148,9 @@ define void @store_nxv6f32(ptr %out) { define void @store_nxv12f16(ptr %out) { ; CHECK-LABEL: store_nxv12f16: ; CHECK: // %bb.0: +; CHECK-NEXT: fmov z0.h, #1.00000000 ; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ptrue p1.h -; CHECK-NEXT: fmov z0.h, #1.00000000 ; CHECK-NEXT: st1h { z0.s }, p0, [x0, #2, mul vl] ; CHECK-NEXT: st1h { z0.h }, p1, [x0] ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sve-stepvector.ll b/llvm/test/CodeGen/AArch64/sve-stepvector.ll index 6f5a31248de7..4c5f27d3e709 100644 --- a/llvm/test/CodeGen/AArch64/sve-stepvector.ll +++ b/llvm/test/CodeGen/AArch64/sve-stepvector.ll @@ -208,9 +208,9 @@ entry: define @multiple_use_stepvector_nxv4i32_1(i32 %data) { ; CHECK-LABEL: multiple_use_stepvector_nxv4i32_1: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: index z0.s, w0, #1 ; CHECK-NEXT: mov z1.s, w0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mul z1.s, p0/m, z1.s, z0.s ; CHECK-NEXT: sub z0.s, z1.s, z0.s ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-bit-counting.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-bit-counting.ll index be5c318e675d..d547f99a0230 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-bit-counting.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-bit-counting.ll @@ -11,8 +11,8 @@ target triple = "aarch64-unknown-linux-gnu" define <4 x i8> @ctlz_v4i8(<4 x i8> %op) { ; CHECK-LABEL: ctlz_v4i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: and z0.h, z0.h, #0xff ; CHECK-NEXT: clz z0.h, p0/m, z0.h ; CHECK-NEXT: sub z0.h, z0.h, #8 // =0x8 @@ -49,8 +49,8 @@ define <16 x i8> @ctlz_v16i8(<16 x i8> %op) { define void @ctlz_v32i8(ptr %a) { ; CHECK-LABEL: ctlz_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: clz z0.b, p0/m, z0.b ; CHECK-NEXT: clz z1.b, p0/m, z1.b ; CHECK-NEXT: stp q0, q1, [x0] @@ -64,8 +64,8 @@ define void @ctlz_v32i8(ptr %a) { define <2 x i16> @ctlz_v2i16(<2 x i16> %op) { ; CHECK-LABEL: ctlz_v2i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: and z0.s, z0.s, #0xffff ; CHECK-NEXT: clz z0.s, p0/m, z0.s ; CHECK-NEXT: sub z0.s, z0.s, #16 // =0x10 @@ -102,8 +102,8 @@ define <8 x i16> @ctlz_v8i16(<8 x i16> %op) { define void @ctlz_v16i16(ptr %a) { ; CHECK-LABEL: ctlz_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: clz z0.h, p0/m, z0.h ; CHECK-NEXT: clz z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x0] @@ -141,8 +141,8 @@ define <4 x i32> @ctlz_v4i32(<4 x i32> %op) { define void @ctlz_v8i32(ptr %a) { ; CHECK-LABEL: ctlz_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: clz z0.s, p0/m, z0.s ; CHECK-NEXT: clz z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -180,8 +180,8 @@ define <2 x i64> @ctlz_v2i64(<2 x i64> %op) { define void @ctlz_v4i64(ptr %a) { ; CHECK-LABEL: ctlz_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: clz z0.d, p0/m, z0.d ; CHECK-NEXT: clz z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] @@ -199,8 +199,8 @@ define void @ctlz_v4i64(ptr %a) { define <4 x i8> @ctpop_v4i8(<4 x i8> %op) { ; CHECK-LABEL: ctpop_v4i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: and z0.h, z0.h, #0xff ; CHECK-NEXT: cnt z0.h, p0/m, z0.h ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 @@ -236,8 +236,8 @@ define <16 x i8> @ctpop_v16i8(<16 x i8> %op) { define void @ctpop_v32i8(ptr %a) { ; CHECK-LABEL: ctpop_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: cnt z0.b, p0/m, z0.b ; CHECK-NEXT: cnt z1.b, p0/m, z1.b ; CHECK-NEXT: stp q0, q1, [x0] @@ -251,8 +251,8 @@ define void @ctpop_v32i8(ptr %a) { define <2 x i16> @ctpop_v2i16(<2 x i16> %op) { ; CHECK-LABEL: ctpop_v2i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: and z0.s, z0.s, #0xffff ; CHECK-NEXT: cnt z0.s, p0/m, z0.s ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 @@ -288,8 +288,8 @@ define <8 x i16> @ctpop_v8i16(<8 x i16> %op) { define void @ctpop_v16i16(ptr %a) { ; CHECK-LABEL: ctpop_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: cnt z0.h, p0/m, z0.h ; CHECK-NEXT: cnt z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x0] @@ -327,8 +327,8 @@ define <4 x i32> @ctpop_v4i32(<4 x i32> %op) { define void @ctpop_v8i32(ptr %a) { ; CHECK-LABEL: ctpop_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: cnt z0.s, p0/m, z0.s ; CHECK-NEXT: cnt z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -366,8 +366,8 @@ define <2 x i64> @ctpop_v2i64(<2 x i64> %op) { define void @ctpop_v4i64(ptr %a) { ; CHECK-LABEL: ctpop_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: cnt z0.d, p0/m, z0.d ; CHECK-NEXT: cnt z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] @@ -385,8 +385,8 @@ define void @ctpop_v4i64(ptr %a) { define <4 x i8> @cttz_v4i8(<4 x i8> %op) { ; CHECK-LABEL: cttz_v4i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: orr z0.h, z0.h, #0x100 ; CHECK-NEXT: rbit z0.h, p0/m, z0.h ; CHECK-NEXT: clz z0.h, p0/m, z0.h @@ -425,8 +425,8 @@ define <16 x i8> @cttz_v16i8(<16 x i8> %op) { define void @cttz_v32i8(ptr %a) { ; CHECK-LABEL: cttz_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: rbit z0.b, p0/m, z0.b ; CHECK-NEXT: rbit z1.b, p0/m, z1.b ; CHECK-NEXT: clz z0.b, p0/m, z0.b @@ -442,8 +442,8 @@ define void @cttz_v32i8(ptr %a) { define <2 x i16> @cttz_v2i16(<2 x i16> %op) { ; CHECK-LABEL: cttz_v2i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: orr z0.s, z0.s, #0x10000 ; CHECK-NEXT: rbit z0.s, p0/m, z0.s ; CHECK-NEXT: clz z0.s, p0/m, z0.s @@ -482,8 +482,8 @@ define <8 x i16> @cttz_v8i16(<8 x i16> %op) { define void @cttz_v16i16(ptr %a) { ; CHECK-LABEL: cttz_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: rbit z0.h, p0/m, z0.h ; CHECK-NEXT: rbit z1.h, p0/m, z1.h ; CHECK-NEXT: clz z0.h, p0/m, z0.h @@ -525,8 +525,8 @@ define <4 x i32> @cttz_v4i32(<4 x i32> %op) { define void @cttz_v8i32(ptr %a) { ; CHECK-LABEL: cttz_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: rbit z0.s, p0/m, z0.s ; CHECK-NEXT: rbit z1.s, p0/m, z1.s ; CHECK-NEXT: clz z0.s, p0/m, z0.s @@ -568,8 +568,8 @@ define <2 x i64> @cttz_v2i64(<2 x i64> %op) { define void @cttz_v4i64(ptr %a) { ; CHECK-LABEL: cttz_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: rbit z0.d, p0/m, z0.d ; CHECK-NEXT: rbit z1.d, p0/m, z1.d ; CHECK-NEXT: clz z0.d, p0/m, z0.d diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ext-loads.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ext-loads.ll index 251a7c3b18a9..0aefba2d4c6a 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ext-loads.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ext-loads.ll @@ -169,15 +169,15 @@ define <16 x i64> @load_zext_v16i16i64(ptr %ap) { ; CHECK-NEXT: ld1h { z1.d }, p0/z, [x0, x8, lsl #1] ; CHECK-NEXT: mov x8, #8 // =0x8 ; CHECK-NEXT: ld1h { z2.d }, p0/z, [x0, x9, lsl #1] -; CHECK-NEXT: mov x9, #10 // =0xa ; CHECK-NEXT: ld1h { z3.d }, p0/z, [x0, x10, lsl #1] -; CHECK-NEXT: mov x10, #12 // =0xc +; CHECK-NEXT: mov x9, #10 // =0xa ; CHECK-NEXT: ld1h { z4.d }, p0/z, [x0, x8, lsl #1] -; CHECK-NEXT: mov x8, #14 // =0xe +; CHECK-NEXT: mov x8, #12 // =0xc +; CHECK-NEXT: mov x10, #14 // =0xe ; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0] ; CHECK-NEXT: ld1h { z5.d }, p0/z, [x0, x9, lsl #1] -; CHECK-NEXT: ld1h { z6.d }, p0/z, [x0, x10, lsl #1] -; CHECK-NEXT: ld1h { z7.d }, p0/z, [x0, x8, lsl #1] +; CHECK-NEXT: ld1h { z6.d }, p0/z, [x0, x8, lsl #1] +; CHECK-NEXT: ld1h { z7.d }, p0/z, [x0, x10, lsl #1] ; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 ; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z1 ; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2 diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll index 2ace0bca274a..0d6675def8b5 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll @@ -241,8 +241,8 @@ define void @test_copysign_v2f32_v2f64(ptr %ap, ptr %bp) { ; SVE-NEXT: ptrue p0.d ; SVE-NEXT: ldr q0, [x1] ; SVE-NEXT: ldr d1, [x0] -; SVE-NEXT: and z1.s, z1.s, #0x7fffffff ; SVE-NEXT: fcvt z0.s, p0/m, z0.d +; SVE-NEXT: and z1.s, z1.s, #0x7fffffff ; SVE-NEXT: uzp1 z0.s, z0.s, z0.s ; SVE-NEXT: and z0.s, z0.s, #0x80000000 ; SVE-NEXT: orr z0.d, z1.d, z0.d @@ -274,8 +274,8 @@ define void @test_copysign_v2f32_v2f64(ptr %ap, ptr %bp) { define void @test_copysign_v4f32_v4f64(ptr %ap, ptr %bp) { ; SVE-LABEL: test_copysign_v4f32_v4f64: ; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.d ; SVE-NEXT: ldp q0, q1, [x1] +; SVE-NEXT: ptrue p0.d ; SVE-NEXT: fcvt z1.s, p0/m, z1.d ; SVE-NEXT: fcvt z0.s, p0/m, z0.d ; SVE-NEXT: ptrue p0.s, vl2 @@ -291,8 +291,8 @@ define void @test_copysign_v4f32_v4f64(ptr %ap, ptr %bp) { ; ; SVE2-LABEL: test_copysign_v4f32_v4f64: ; SVE2: // %bb.0: -; SVE2-NEXT: ptrue p0.d ; SVE2-NEXT: ldp q0, q1, [x1] +; SVE2-NEXT: ptrue p0.d ; SVE2-NEXT: ldr q2, [x0] ; SVE2-NEXT: fcvt z1.s, p0/m, z1.d ; SVE2-NEXT: fcvt z0.s, p0/m, z0.d @@ -319,8 +319,8 @@ define void @test_copysign_v2f64_v2f32(ptr %ap, ptr %bp) { ; SVE: // %bb.0: ; SVE-NEXT: ptrue p0.d, vl2 ; SVE-NEXT: ldr q0, [x0] -; SVE-NEXT: and z0.d, z0.d, #0x7fffffffffffffff ; SVE-NEXT: ld1w { z1.d }, p0/z, [x1] +; SVE-NEXT: and z0.d, z0.d, #0x7fffffffffffffff ; SVE-NEXT: fcvt z1.d, p0/m, z1.s ; SVE-NEXT: and z1.d, z1.d, #0x8000000000000000 ; SVE-NEXT: orr z0.d, z0.d, z1.d @@ -354,10 +354,10 @@ define void @test_copysign_v4f64_v4f32(ptr %ap, ptr %bp) { ; SVE-NEXT: ptrue p0.d, vl2 ; SVE-NEXT: mov x8, #2 // =0x2 ; SVE-NEXT: ldp q2, q3, [x0] -; SVE-NEXT: and z2.d, z2.d, #0x7fffffffffffffff -; SVE-NEXT: and z3.d, z3.d, #0x7fffffffffffffff ; SVE-NEXT: ld1w { z0.d }, p0/z, [x1] ; SVE-NEXT: ld1w { z1.d }, p0/z, [x1, x8, lsl #2] +; SVE-NEXT: and z2.d, z2.d, #0x7fffffffffffffff +; SVE-NEXT: and z3.d, z3.d, #0x7fffffffffffffff ; SVE-NEXT: fcvt z0.d, p0/m, z0.s ; SVE-NEXT: fcvt z1.d, p0/m, z1.s ; SVE-NEXT: and z0.d, z0.d, #0x8000000000000000 @@ -397,8 +397,8 @@ define void @test_copysign_v4f16_v4f32(ptr %ap, ptr %bp) { ; SVE-NEXT: ptrue p0.s ; SVE-NEXT: ldr q0, [x1] ; SVE-NEXT: ldr d1, [x0] -; SVE-NEXT: and z1.h, z1.h, #0x7fff ; SVE-NEXT: fcvt z0.h, p0/m, z0.s +; SVE-NEXT: and z1.h, z1.h, #0x7fff ; SVE-NEXT: uzp1 z0.h, z0.h, z0.h ; SVE-NEXT: and z0.h, z0.h, #0x8000 ; SVE-NEXT: orr z0.d, z1.d, z0.d @@ -429,13 +429,13 @@ define void @test_copysign_v4f16_v4f64(ptr %ap, ptr %bp) { ; SVE: // %bb.0: ; SVE-NEXT: ldp q0, q1, [x1] ; SVE-NEXT: ptrue p0.s, vl2 -; SVE-NEXT: ptrue p1.s ; SVE-NEXT: fcvtxn v1.2s, v1.2d ; SVE-NEXT: fcvtxn v0.2s, v0.2d ; SVE-NEXT: splice z0.s, p0, z0.s, z1.s +; SVE-NEXT: ptrue p0.s ; SVE-NEXT: ldr d1, [x0] ; SVE-NEXT: and z1.h, z1.h, #0x7fff -; SVE-NEXT: fcvt z0.h, p1/m, z0.s +; SVE-NEXT: fcvt z0.h, p0/m, z0.s ; SVE-NEXT: uzp1 z0.h, z0.h, z0.h ; SVE-NEXT: and z0.h, z0.h, #0x8000 ; SVE-NEXT: orr z0.d, z1.d, z0.d @@ -446,13 +446,13 @@ define void @test_copysign_v4f16_v4f64(ptr %ap, ptr %bp) { ; SVE2: // %bb.0: ; SVE2-NEXT: ldp q0, q1, [x1] ; SVE2-NEXT: ptrue p0.s, vl2 -; SVE2-NEXT: ptrue p1.s ; SVE2-NEXT: ldr d2, [x0] ; SVE2-NEXT: fcvtxn v1.2s, v1.2d ; SVE2-NEXT: fcvtxn v0.2s, v0.2d ; SVE2-NEXT: splice z0.s, p0, z0.s, z1.s +; SVE2-NEXT: ptrue p0.s ; SVE2-NEXT: mov z1.h, #32767 // =0x7fff -; SVE2-NEXT: fcvt z0.h, p1/m, z0.s +; SVE2-NEXT: fcvt z0.h, p0/m, z0.s ; SVE2-NEXT: uzp1 z0.h, z0.h, z0.h ; SVE2-NEXT: bsl z2.d, z2.d, z0.d, z1.d ; SVE2-NEXT: str d2, [x0] @@ -470,8 +470,8 @@ define void @test_copysign_v4f16_v4f64(ptr %ap, ptr %bp) { define void @test_copysign_v8f16_v8f32(ptr %ap, ptr %bp) { ; SVE-LABEL: test_copysign_v8f16_v8f32: ; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.s ; SVE-NEXT: ldp q0, q1, [x1] +; SVE-NEXT: ptrue p0.s ; SVE-NEXT: fcvt z1.h, p0/m, z1.s ; SVE-NEXT: fcvt z0.h, p0/m, z0.s ; SVE-NEXT: ptrue p0.h, vl4 @@ -487,8 +487,8 @@ define void @test_copysign_v8f16_v8f32(ptr %ap, ptr %bp) { ; ; SVE2-LABEL: test_copysign_v8f16_v8f32: ; SVE2: // %bb.0: -; SVE2-NEXT: ptrue p0.s ; SVE2-NEXT: ldp q0, q1, [x1] +; SVE2-NEXT: ptrue p0.s ; SVE2-NEXT: ldr q2, [x0] ; SVE2-NEXT: fcvt z1.h, p0/m, z1.s ; SVE2-NEXT: fcvt z0.h, p0/m, z0.s diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-arith.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-arith.ll index c436dea8ff1b..c2d6ed4e9ccf 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-arith.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-arith.ll @@ -50,8 +50,8 @@ define <8 x half> @fadd_v8f16(<8 x half> %op1, <8 x half> %op2) { define void @fadd_v16f16(ptr %a, ptr %b) { ; CHECK-LABEL: fadd_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fadd z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -94,8 +94,8 @@ define <4 x float> @fadd_v4f32(<4 x float> %op1, <4 x float> %op2) { define void @fadd_v8f32(ptr %a, ptr %b) { ; CHECK-LABEL: fadd_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fadd z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -125,8 +125,8 @@ define <2 x double> @fadd_v2f64(<2 x double> %op1, <2 x double> %op2) { define void @fadd_v4f64(ptr %a, ptr %b) { ; CHECK-LABEL: fadd_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fadd z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 @@ -186,8 +186,8 @@ define <8 x half> @fdiv_v8f16(<8 x half> %op1, <8 x half> %op2) { define void @fdiv_v16f16(ptr %a, ptr %b) { ; CHECK-LABEL: fdiv_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fdivr z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -230,8 +230,8 @@ define <4 x float> @fdiv_v4f32(<4 x float> %op1, <4 x float> %op2) { define void @fdiv_v8f32(ptr %a, ptr %b) { ; CHECK-LABEL: fdiv_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fdivr z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -261,8 +261,8 @@ define <2 x double> @fdiv_v2f64(<2 x double> %op1, <2 x double> %op2) { define void @fdiv_v4f64(ptr %a, ptr %b) { ; CHECK-LABEL: fdiv_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fdivr z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 @@ -325,8 +325,8 @@ define <8 x half> @fma_v8f16(<8 x half> %op1, <8 x half> %op2, <8 x half> %op3) define void @fma_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fma_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q4, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q5, [x2] ; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: fmad z0.h, p0/m, z2.h, z1.h @@ -373,8 +373,8 @@ define <4 x float> @fma_v4f32(<4 x float> %op1, <4 x float> %op2, <4 x float> %o define void @fma_v8f32(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fma_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q4, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q5, [x2] ; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: fmad z0.s, p0/m, z2.s, z1.s @@ -407,8 +407,8 @@ define <2 x double> @fma_v2f64(<2 x double> %op1, <2 x double> %op2, <2 x double define void @fma_v4f64(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fma_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q4, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q5, [x2] ; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: fmad z0.d, p0/m, z2.d, z1.d @@ -470,8 +470,8 @@ define <8 x half> @fmul_v8f16(<8 x half> %op1, <8 x half> %op2) { define void @fmul_v16f16(ptr %a, ptr %b) { ; CHECK-LABEL: fmul_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fmul z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -514,8 +514,8 @@ define <4 x float> @fmul_v4f32(<4 x float> %op1, <4 x float> %op2) { define void @fmul_v8f32(ptr %a, ptr %b) { ; CHECK-LABEL: fmul_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fmul z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -545,8 +545,8 @@ define <2 x double> @fmul_v2f64(<2 x double> %op1, <2 x double> %op2) { define void @fmul_v4f64(ptr %a, ptr %b) { ; CHECK-LABEL: fmul_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fmul z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 @@ -603,8 +603,8 @@ define <8 x half> @fneg_v8f16(<8 x half> %op) { define void @fneg_v16f16(ptr %a, ptr %b) { ; CHECK-LABEL: fneg_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: fneg z0.h, p0/m, z0.h ; CHECK-NEXT: fneg z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x0] @@ -642,8 +642,8 @@ define <4 x float> @fneg_v4f32(<4 x float> %op) { define void @fneg_v8f32(ptr %a) { ; CHECK-LABEL: fneg_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: fneg z0.s, p0/m, z0.s ; CHECK-NEXT: fneg z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -669,8 +669,8 @@ define <2 x double> @fneg_v2f64(<2 x double> %op) { define void @fneg_v4f64(ptr %a) { ; CHECK-LABEL: fneg_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: fneg z0.d, p0/m, z0.d ; CHECK-NEXT: fneg z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] @@ -724,8 +724,8 @@ define <8 x half> @fsqrt_v8f16(<8 x half> %op) { define void @fsqrt_v16f16(ptr %a, ptr %b) { ; CHECK-LABEL: fsqrt_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: fsqrt z0.h, p0/m, z0.h ; CHECK-NEXT: fsqrt z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x0] @@ -763,8 +763,8 @@ define <4 x float> @fsqrt_v4f32(<4 x float> %op) { define void @fsqrt_v8f32(ptr %a) { ; CHECK-LABEL: fsqrt_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: fsqrt z0.s, p0/m, z0.s ; CHECK-NEXT: fsqrt z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -790,8 +790,8 @@ define <2 x double> @fsqrt_v2f64(<2 x double> %op) { define void @fsqrt_v4f64(ptr %a) { ; CHECK-LABEL: fsqrt_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: fsqrt z0.d, p0/m, z0.d ; CHECK-NEXT: fsqrt z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] @@ -848,8 +848,8 @@ define <8 x half> @fsub_v8f16(<8 x half> %op1, <8 x half> %op2) { define void @fsub_v16f16(ptr %a, ptr %b) { ; CHECK-LABEL: fsub_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fsubr z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -892,8 +892,8 @@ define <4 x float> @fsub_v4f32(<4 x float> %op1, <4 x float> %op2) { define void @fsub_v8f32(ptr %a, ptr %b) { ; CHECK-LABEL: fsub_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fsubr z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -923,8 +923,8 @@ define <2 x double> @fsub_v2f64(<2 x double> %op1, <2 x double> %op2) { define void @fsub_v4f64(ptr %a, ptr %b) { ; CHECK-LABEL: fsub_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fsubr z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 @@ -981,8 +981,8 @@ define <8 x half> @fabs_v8f16(<8 x half> %op) { define void @fabs_v16f16(ptr %a) { ; CHECK-LABEL: fabs_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: fabs z0.h, p0/m, z0.h ; CHECK-NEXT: fabs z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x0] @@ -1020,8 +1020,8 @@ define <4 x float> @fabs_v4f32(<4 x float> %op) { define void @fabs_v8f32(ptr %a) { ; CHECK-LABEL: fabs_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: fabs z0.s, p0/m, z0.s ; CHECK-NEXT: fabs z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -1047,8 +1047,8 @@ define <2 x double> @fabs_v2f64(<2 x double> %op) { define void @fabs_v4f64(ptr %a) { ; CHECK-LABEL: fabs_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: fabs z0.d, p0/m, z0.d ; CHECK-NEXT: fabs z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-compares.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-compares.ll index aad078f035f7..e92694d1fc80 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-compares.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-compares.ll @@ -57,8 +57,8 @@ define <8 x i16> @fcmp_oeq_v8f16(<8 x half> %op1, <8 x half> %op2) { define void @fcmp_oeq_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_oeq_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmeq p1.h, p0/z, z1.h, z0.h ; CHECK-NEXT: fcmeq p0.h, p0/z, z2.h, z3.h @@ -107,8 +107,8 @@ define <4 x i32> @fcmp_oeq_v4f32(<4 x float> %op1, <4 x float> %op2) { define void @fcmp_oeq_v8f32(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_oeq_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmeq p1.s, p0/z, z1.s, z0.s ; CHECK-NEXT: fcmeq p0.s, p0/z, z2.s, z3.s @@ -157,8 +157,8 @@ define <2 x i64> @fcmp_oeq_v2f64(<2 x double> %op1, <2 x double> %op2) { define void @fcmp_oeq_v4f64(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_oeq_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmeq p1.d, p0/z, z1.d, z0.d ; CHECK-NEXT: fcmeq p0.d, p0/z, z2.d, z3.d @@ -181,8 +181,8 @@ define void @fcmp_oeq_v4f64(ptr %a, ptr %b, ptr %c) { define void @fcmp_ueq_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_ueq_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmuo p1.h, p0/z, z1.h, z0.h ; CHECK-NEXT: fcmeq p2.h, p0/z, z1.h, z0.h @@ -209,8 +209,8 @@ define void @fcmp_ueq_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_one_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_one_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmgt p1.h, p0/z, z0.h, z1.h ; CHECK-NEXT: fcmgt p2.h, p0/z, z1.h, z0.h @@ -237,8 +237,8 @@ define void @fcmp_one_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_une_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_une_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmne p1.h, p0/z, z1.h, z0.h ; CHECK-NEXT: fcmne p0.h, p0/z, z2.h, z3.h @@ -261,8 +261,8 @@ define void @fcmp_une_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_ogt_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_ogt_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmgt p1.h, p0/z, z1.h, z0.h ; CHECK-NEXT: fcmgt p0.h, p0/z, z2.h, z3.h @@ -285,8 +285,8 @@ define void @fcmp_ogt_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_ugt_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_ugt_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmge p1.h, p0/z, z0.h, z1.h ; CHECK-NEXT: fcmge p0.h, p0/z, z3.h, z2.h @@ -312,8 +312,8 @@ define void @fcmp_ugt_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_olt_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_olt_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmgt p1.h, p0/z, z0.h, z1.h ; CHECK-NEXT: fcmgt p0.h, p0/z, z3.h, z2.h @@ -336,8 +336,8 @@ define void @fcmp_olt_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_ult_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_ult_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmge p1.h, p0/z, z1.h, z0.h ; CHECK-NEXT: fcmge p0.h, p0/z, z2.h, z3.h @@ -363,8 +363,8 @@ define void @fcmp_ult_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_oge_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_oge_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmge p1.h, p0/z, z1.h, z0.h ; CHECK-NEXT: fcmge p0.h, p0/z, z2.h, z3.h @@ -387,8 +387,8 @@ define void @fcmp_oge_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_uge_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_uge_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmgt p1.h, p0/z, z0.h, z1.h ; CHECK-NEXT: fcmgt p0.h, p0/z, z3.h, z2.h @@ -414,8 +414,8 @@ define void @fcmp_uge_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_ole_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_ole_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmge p1.h, p0/z, z0.h, z1.h ; CHECK-NEXT: fcmge p0.h, p0/z, z3.h, z2.h @@ -438,8 +438,8 @@ define void @fcmp_ole_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_ule_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_ule_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmgt p1.h, p0/z, z1.h, z0.h ; CHECK-NEXT: fcmgt p0.h, p0/z, z2.h, z3.h @@ -465,8 +465,8 @@ define void @fcmp_ule_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_uno_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_uno_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmuo p1.h, p0/z, z1.h, z0.h ; CHECK-NEXT: fcmuo p0.h, p0/z, z2.h, z3.h @@ -489,8 +489,8 @@ define void @fcmp_uno_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_ord_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_ord_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmuo p1.h, p0/z, z1.h, z0.h ; CHECK-NEXT: fcmuo p0.h, p0/z, z2.h, z3.h @@ -516,8 +516,8 @@ define void @fcmp_ord_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_eq_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_eq_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmeq p1.h, p0/z, z1.h, z0.h ; CHECK-NEXT: fcmeq p0.h, p0/z, z2.h, z3.h @@ -540,8 +540,8 @@ define void @fcmp_eq_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_ne_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_ne_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmne p1.h, p0/z, z1.h, z0.h ; CHECK-NEXT: fcmne p0.h, p0/z, z2.h, z3.h @@ -564,8 +564,8 @@ define void @fcmp_ne_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_gt_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_gt_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmgt p1.h, p0/z, z1.h, z0.h ; CHECK-NEXT: fcmgt p0.h, p0/z, z2.h, z3.h @@ -588,8 +588,8 @@ define void @fcmp_gt_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_lt_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_lt_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmgt p1.h, p0/z, z0.h, z1.h ; CHECK-NEXT: fcmgt p0.h, p0/z, z3.h, z2.h @@ -612,8 +612,8 @@ define void @fcmp_lt_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_ge_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_ge_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmge p1.h, p0/z, z1.h, z0.h ; CHECK-NEXT: fcmge p0.h, p0/z, z2.h, z3.h @@ -636,8 +636,8 @@ define void @fcmp_ge_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_le_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_le_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmge p1.h, p0/z, z0.h, z1.h ; CHECK-NEXT: fcmge p0.h, p0/z, z3.h, z2.h diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-convert.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-convert.ll index 18f9a4d371d0..9bdde14e8d83 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-convert.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-convert.ll @@ -8,9 +8,9 @@ target triple = "aarch64-unknown-linux-gnu" define void @fp_convert_combine_crash(ptr %a, ptr %b) { ; CHECK-LABEL: fp_convert_combine_crash: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fmov z0.s, #8.00000000 +; CHECK-NEXT: ldp q1, q2, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: fmul z1.s, p0/m, z1.s, z0.s ; CHECK-NEXT: fmul z0.s, p0/m, z0.s, z2.s ; CHECK-NEXT: fcvtzs z1.s, p0/m, z1.s diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-extend-trunc.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-extend-trunc.ll index 28e02da53af4..244a40510173 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-extend-trunc.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-extend-trunc.ll @@ -11,8 +11,8 @@ target triple = "aarch64-unknown-linux-gnu" define void @fcvt_v2f16_to_v2f32(<2 x half> %a, ptr %b) { ; CHECK-LABEL: fcvt_v2f16_to_v2f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: fcvt z0.s, p0/m, z0.h ; CHECK-NEXT: str d0, [x0] @@ -25,8 +25,8 @@ define void @fcvt_v2f16_to_v2f32(<2 x half> %a, ptr %b) { define void @fcvt_v4f16_to_v4f32(<4 x half> %a, ptr %b) { ; CHECK-LABEL: fcvt_v4f16_to_v4f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: fcvt z0.s, p0/m, z0.h ; CHECK-NEXT: str q0, [x0] @@ -371,8 +371,8 @@ define void @fcvt_v4f32_v4f16(ptr %a, ptr %b) { define void @fcvt_v8f32_v8f16(ptr %a, ptr %b) { ; CHECK-LABEL: fcvt_v8f32_v8f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: mov x8, #4 // =0x4 ; CHECK-NEXT: fcvt z0.h, p0/m, z0.s ; CHECK-NEXT: fcvt z1.h, p0/m, z1.s @@ -420,8 +420,8 @@ define void @fcvt_v2f64_v2f16(ptr %a, ptr %b) { define void @fcvt_v4f64_v4f16(ptr %a, ptr %b) { ; CHECK-LABEL: fcvt_v4f64_v4f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: mov x8, #2 // =0x2 ; CHECK-NEXT: fcvt z0.h, p0/m, z0.d ; CHECK-NEXT: fcvt z1.h, p0/m, z1.d @@ -467,8 +467,8 @@ define void @fcvt_v2f64_v2f32(<2 x double> %op1, ptr %b) { define void @fcvt_v4f64_v4f32(ptr %a, ptr %b) { ; CHECK-LABEL: fcvt_v4f64_v4f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: mov x8, #2 // =0x2 ; CHECK-NEXT: fcvt z0.s, p0/m, z0.d ; CHECK-NEXT: fcvt z1.s, p0/m, z1.d diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-fma.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-fma.ll index b5df97f767c1..478be9ab76dd 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-fma.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-fma.ll @@ -40,8 +40,8 @@ define <8 x half> @fma_v8f16(<8 x half> %op1, <8 x half> %op2, <8 x half> %op3) define void @fma_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fma_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q4, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q5, [x2] ; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: fmad z0.h, p0/m, z2.h, z1.h @@ -91,8 +91,8 @@ define <4 x float> @fma_v4f32(<4 x float> %op1, <4 x float> %op2, <4 x float> %o define void @fma_v8f32(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fma_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q4, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q5, [x2] ; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: fmad z0.s, p0/m, z2.s, z1.s @@ -140,8 +140,8 @@ define <2 x double> @fma_v2f64(<2 x double> %op1, <2 x double> %op2, <2 x double define void @fma_v4f64(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fma_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q4, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q5, [x2] ; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: fmad z0.d, p0/m, z2.d, z1.d diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-minmax.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-minmax.ll index 07a67e265029..4dc034adf459 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-minmax.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-minmax.ll @@ -37,8 +37,8 @@ define <8 x half> @fmaxnm_v8f16(<8 x half> %op1, <8 x half> %op2) { define void @fmaxnm_v16f16(ptr %a, ptr %b) { ; CHECK-LABEL: fmaxnm_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fmaxnm z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -81,8 +81,8 @@ define <4 x float> @fmaxnm_v4f32(<4 x float> %op1, <4 x float> %op2) { define void @fmaxnm_v8f32(ptr %a, ptr %b) { ; CHECK-LABEL: fmaxnm_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fmaxnm z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -123,8 +123,8 @@ define <2 x double> @fmaxnm_v2f64(<2 x double> %op1, <2 x double> %op2) { define void @fmaxnm_v4f64(ptr %a, ptr %b) { ; CHECK-LABEL: fmaxnm_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fmaxnm z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 @@ -171,8 +171,8 @@ define <8 x half> @fminnm_v8f16(<8 x half> %op1, <8 x half> %op2) { define void @fminnm_v16f16(ptr %a, ptr %b) { ; CHECK-LABEL: fminnm_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fminnm z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -215,8 +215,8 @@ define <4 x float> @fminnm_v4f32(<4 x float> %op1, <4 x float> %op2) { define void @fminnm_v8f32(ptr %a, ptr %b) { ; CHECK-LABEL: fminnm_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fminnm z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -257,8 +257,8 @@ define <2 x double> @fminnm_v2f64(<2 x double> %op1, <2 x double> %op2) { define void @fminnm_v4f64(ptr %a, ptr %b) { ; CHECK-LABEL: fminnm_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fminnm z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 @@ -305,8 +305,8 @@ define <8 x half> @fmax_v8f16(<8 x half> %op1, <8 x half> %op2) { define void @fmax_v16f16(ptr %a, ptr %b) { ; CHECK-LABEL: fmax_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fmax z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -349,8 +349,8 @@ define <4 x float> @fmax_v4f32(<4 x float> %op1, <4 x float> %op2) { define void @fmax_v8f32(ptr %a, ptr %b) { ; CHECK-LABEL: fmax_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fmax z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -391,8 +391,8 @@ define <2 x double> @fmax_v2f64(<2 x double> %op1, <2 x double> %op2) { define void @fmax_v4f64(ptr %a, ptr %b) { ; CHECK-LABEL: fmax_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fmax z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 @@ -439,8 +439,8 @@ define <8 x half> @fmin_v8f16(<8 x half> %op1, <8 x half> %op2) { define void @fmin_v16f16(ptr %a, ptr %b) { ; CHECK-LABEL: fmin_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fmin z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -483,8 +483,8 @@ define <4 x float> @fmin_v4f32(<4 x float> %op1, <4 x float> %op2) { define void @fmin_v8f32(ptr %a, ptr %b) { ; CHECK-LABEL: fmin_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fmin z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -525,8 +525,8 @@ define <2 x double> @fmin_v2f64(<2 x double> %op1, <2 x double> %op2) { define void @fmin_v4f64(ptr %a, ptr %b) { ; CHECK-LABEL: fmin_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fmin z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-reduce.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-reduce.ll index d2d771c48c20..bd10a0e091c0 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-reduce.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-reduce.ll @@ -211,8 +211,8 @@ define half @faddv_v8f16(half %start, <8 x half> %a) { define half @faddv_v16f16(half %start, ptr %a) { ; CHECK-LABEL: faddv_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q2, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: fadd z1.h, p0/m, z1.h, z2.h ; CHECK-NEXT: faddv h1, p0, z1.h ; CHECK-NEXT: fadd h0, h0, h1 @@ -249,8 +249,8 @@ define float @faddv_v4f32(float %start, <4 x float> %a) { define float @faddv_v8f32(float %start, ptr %a) { ; CHECK-LABEL: faddv_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q2, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: fadd z1.s, p0/m, z1.s, z2.s ; CHECK-NEXT: faddv s1, p0, z1.s ; CHECK-NEXT: fadd s0, s0, s1 @@ -285,8 +285,8 @@ define double @faddv_v2f64(double %start, <2 x double> %a) { define double @faddv_v4f64(double %start, ptr %a) { ; CHECK-LABEL: faddv_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q2, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: fadd z1.d, p0/m, z1.d, z2.d ; CHECK-NEXT: faddv d1, p0, z1.d ; CHECK-NEXT: fadd d0, d0, d1 @@ -327,8 +327,8 @@ define half @fmaxv_v8f16(<8 x half> %a) { define half @fmaxv_v16f16(ptr %a) { ; CHECK-LABEL: fmaxv_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: fmaxnm z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: fmaxnmv h0, p0, z0.h ; CHECK-NEXT: // kill: def $h0 killed $h0 killed $z0 @@ -365,8 +365,8 @@ define float @fmaxv_v4f32(<4 x float> %a) { define float @fmaxv_v8f32(ptr %a) { ; CHECK-LABEL: fmaxv_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: fmaxnm z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: fmaxnmv s0, p0, z0.s ; CHECK-NEXT: // kill: def $s0 killed $s0 killed $z0 @@ -401,8 +401,8 @@ define double @fmaxv_v2f64(<2 x double> %a) { define double @fmaxv_v4f64(ptr %a) { ; CHECK-LABEL: fmaxv_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: fmaxnm z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: fmaxnmv d0, p0, z0.d ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 @@ -443,8 +443,8 @@ define half @fminv_v8f16(<8 x half> %a) { define half @fminv_v16f16(ptr %a) { ; CHECK-LABEL: fminv_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: fminnm z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: fminnmv h0, p0, z0.h ; CHECK-NEXT: // kill: def $h0 killed $h0 killed $z0 @@ -481,8 +481,8 @@ define float @fminv_v4f32(<4 x float> %a) { define float @fminv_v8f32(ptr %a) { ; CHECK-LABEL: fminv_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: fminnm z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: fminnmv s0, p0, z0.s ; CHECK-NEXT: // kill: def $s0 killed $s0 killed $z0 @@ -517,8 +517,8 @@ define double @fminv_v2f64(<2 x double> %a) { define double @fminv_v4f64(ptr %a) { ; CHECK-LABEL: fminv_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: fminnm z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: fminnmv d0, p0, z0.d ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 @@ -559,8 +559,8 @@ define half @fmaximumv_v8f16(<8 x half> %a) { define half @fmaximumv_v16f16(ptr %a) { ; CHECK-LABEL: fmaximumv_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: fmax z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: fmaxv h0, p0, z0.h ; CHECK-NEXT: // kill: def $h0 killed $h0 killed $z0 @@ -597,8 +597,8 @@ define float @fmaximumv_v4f32(<4 x float> %a) { define float @fmaximumv_v8f32(ptr %a) { ; CHECK-LABEL: fmaximumv_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: fmax z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: fmaxv s0, p0, z0.s ; CHECK-NEXT: // kill: def $s0 killed $s0 killed $z0 @@ -633,8 +633,8 @@ define double @fmaximumv_v2f64(<2 x double> %a) { define double @fmaximumv_v4f64(ptr %a) { ; CHECK-LABEL: fmaximumv_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: fmax z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: fmaxv d0, p0, z0.d ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 @@ -675,8 +675,8 @@ define half @fminimumv_v8f16(<8 x half> %a) { define half @fminimumv_v16f16(ptr %a) { ; CHECK-LABEL: fminimumv_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: fmin z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: fminv h0, p0, z0.h ; CHECK-NEXT: // kill: def $h0 killed $h0 killed $z0 @@ -713,8 +713,8 @@ define float @fminimumv_v4f32(<4 x float> %a) { define float @fminimumv_v8f32(ptr %a) { ; CHECK-LABEL: fminimumv_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: fmin z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: fminv s0, p0, z0.s ; CHECK-NEXT: // kill: def $s0 killed $s0 killed $z0 @@ -749,8 +749,8 @@ define double @fminimumv_v2f64(<2 x double> %a) { define double @fminimumv_v4f64(ptr %a) { ; CHECK-LABEL: fminimumv_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: fmin z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: fminv d0, p0, z0.d ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-rounding.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-rounding.ll index 580b43531070..24832d807c64 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-rounding.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-rounding.ll @@ -47,8 +47,8 @@ define <8 x half> @frintp_v8f16(<8 x half> %op) { define void @frintp_v16f16(ptr %a) { ; CHECK-LABEL: frintp_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: frintp z0.h, p0/m, z0.h ; CHECK-NEXT: frintp z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x0] @@ -86,8 +86,8 @@ define <4 x float> @frintp_v4f32(<4 x float> %op) { define void @frintp_v8f32(ptr %a) { ; CHECK-LABEL: frintp_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: frintp z0.s, p0/m, z0.s ; CHECK-NEXT: frintp z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -123,8 +123,8 @@ define <2 x double> @frintp_v2f64(<2 x double> %op) { define void @frintp_v4f64(ptr %a) { ; CHECK-LABEL: frintp_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: frintp z0.d, p0/m, z0.d ; CHECK-NEXT: frintp z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] @@ -178,8 +178,8 @@ define <8 x half> @frintm_v8f16(<8 x half> %op) { define void @frintm_v16f16(ptr %a) { ; CHECK-LABEL: frintm_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: frintm z0.h, p0/m, z0.h ; CHECK-NEXT: frintm z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x0] @@ -217,8 +217,8 @@ define <4 x float> @frintm_v4f32(<4 x float> %op) { define void @frintm_v8f32(ptr %a) { ; CHECK-LABEL: frintm_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: frintm z0.s, p0/m, z0.s ; CHECK-NEXT: frintm z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -254,8 +254,8 @@ define <2 x double> @frintm_v2f64(<2 x double> %op) { define void @frintm_v4f64(ptr %a) { ; CHECK-LABEL: frintm_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: frintm z0.d, p0/m, z0.d ; CHECK-NEXT: frintm z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] @@ -309,8 +309,8 @@ define <8 x half> @frinti_v8f16(<8 x half> %op) { define void @frinti_v16f16(ptr %a) { ; CHECK-LABEL: frinti_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: frinti z0.h, p0/m, z0.h ; CHECK-NEXT: frinti z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x0] @@ -348,8 +348,8 @@ define <4 x float> @frinti_v4f32(<4 x float> %op) { define void @frinti_v8f32(ptr %a) { ; CHECK-LABEL: frinti_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: frinti z0.s, p0/m, z0.s ; CHECK-NEXT: frinti z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -385,8 +385,8 @@ define <2 x double> @frinti_v2f64(<2 x double> %op) { define void @frinti_v4f64(ptr %a) { ; CHECK-LABEL: frinti_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: frinti z0.d, p0/m, z0.d ; CHECK-NEXT: frinti z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] @@ -440,8 +440,8 @@ define <8 x half> @frintx_v8f16(<8 x half> %op) { define void @frintx_v16f16(ptr %a) { ; CHECK-LABEL: frintx_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: frintx z0.h, p0/m, z0.h ; CHECK-NEXT: frintx z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x0] @@ -479,8 +479,8 @@ define <4 x float> @frintx_v4f32(<4 x float> %op) { define void @frintx_v8f32(ptr %a) { ; CHECK-LABEL: frintx_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: frintx z0.s, p0/m, z0.s ; CHECK-NEXT: frintx z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -516,8 +516,8 @@ define <2 x double> @frintx_v2f64(<2 x double> %op) { define void @frintx_v4f64(ptr %a) { ; CHECK-LABEL: frintx_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: frintx z0.d, p0/m, z0.d ; CHECK-NEXT: frintx z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] @@ -571,8 +571,8 @@ define <8 x half> @frinta_v8f16(<8 x half> %op) { define void @frinta_v16f16(ptr %a) { ; CHECK-LABEL: frinta_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: frinta z0.h, p0/m, z0.h ; CHECK-NEXT: frinta z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x0] @@ -610,8 +610,8 @@ define <4 x float> @frinta_v4f32(<4 x float> %op) { define void @frinta_v8f32(ptr %a) { ; CHECK-LABEL: frinta_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: frinta z0.s, p0/m, z0.s ; CHECK-NEXT: frinta z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -647,8 +647,8 @@ define <2 x double> @frinta_v2f64(<2 x double> %op) { define void @frinta_v4f64(ptr %a) { ; CHECK-LABEL: frinta_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: frinta z0.d, p0/m, z0.d ; CHECK-NEXT: frinta z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] @@ -702,8 +702,8 @@ define <8 x half> @frintn_v8f16(<8 x half> %op) { define void @frintn_v16f16(ptr %a) { ; CHECK-LABEL: frintn_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: frintn z0.h, p0/m, z0.h ; CHECK-NEXT: frintn z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x0] @@ -741,8 +741,8 @@ define <4 x float> @frintn_v4f32(<4 x float> %op) { define void @frintn_v8f32(ptr %a) { ; CHECK-LABEL: frintn_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: frintn z0.s, p0/m, z0.s ; CHECK-NEXT: frintn z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -778,8 +778,8 @@ define <2 x double> @frintn_v2f64(<2 x double> %op) { define void @frintn_v4f64(ptr %a) { ; CHECK-LABEL: frintn_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: frintn z0.d, p0/m, z0.d ; CHECK-NEXT: frintn z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] @@ -833,8 +833,8 @@ define <8 x half> @frintz_v8f16(<8 x half> %op) { define void @frintz_v16f16(ptr %a) { ; CHECK-LABEL: frintz_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: frintz z0.h, p0/m, z0.h ; CHECK-NEXT: frintz z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x0] @@ -872,8 +872,8 @@ define <4 x float> @frintz_v4f32(<4 x float> %op) { define void @frintz_v8f32(ptr %a) { ; CHECK-LABEL: frintz_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: frintz z0.s, p0/m, z0.s ; CHECK-NEXT: frintz z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -909,8 +909,8 @@ define <2 x double> @frintz_v2f64(<2 x double> %op) { define void @frintz_v4f64(ptr %a) { ; CHECK-LABEL: frintz_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: frintz z0.d, p0/m, z0.d ; CHECK-NEXT: frintz z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-select.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-select.ll index 73fd7e146534..132225546fc4 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-select.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-select.ll @@ -7,8 +7,8 @@ target triple = "aarch64-unknown-linux-gnu" define <2 x half> @select_v2f16(<2 x half> %op1, <2 x half> %op2, i1 %mask) { ; CHECK-LABEL: select_v2f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w0 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 ; CHECK-NEXT: and z2.h, z2.h, #0x1 @@ -23,8 +23,8 @@ define <2 x half> @select_v2f16(<2 x half> %op1, <2 x half> %op2, i1 %mask) { define <4 x half> @select_v4f16(<4 x half> %op1, <4 x half> %op2, i1 %mask) { ; CHECK-LABEL: select_v4f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w0 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 ; CHECK-NEXT: and z2.h, z2.h, #0x1 @@ -39,8 +39,8 @@ define <4 x half> @select_v4f16(<4 x half> %op1, <4 x half> %op2, i1 %mask) { define <8 x half> @select_v8f16(<8 x half> %op1, <8 x half> %op2, i1 %mask) { ; CHECK-LABEL: select_v8f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w0 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 ; CHECK-NEXT: and z2.h, z2.h, #0x1 @@ -55,8 +55,8 @@ define <8 x half> @select_v8f16(<8 x half> %op1, <8 x half> %op2, i1 %mask) { define void @select_v16f16(ptr %a, ptr %b, i1 %mask) { ; CHECK-LABEL: select_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z0.h, w2 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: and z0.h, z0.h, #0x1 ; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #0 ; CHECK-NEXT: ldr q0, [x0] @@ -77,8 +77,8 @@ define void @select_v16f16(ptr %a, ptr %b, i1 %mask) { define <2 x float> @select_v2f32(<2 x float> %op1, <2 x float> %op2, i1 %mask) { ; CHECK-LABEL: select_v2f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and w8, w0, #0x1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 ; CHECK-NEXT: mov z2.s, w8 @@ -93,8 +93,8 @@ define <2 x float> @select_v2f32(<2 x float> %op1, <2 x float> %op2, i1 %mask) { define <4 x float> @select_v4f32(<4 x float> %op1, <4 x float> %op2, i1 %mask) { ; CHECK-LABEL: select_v4f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and w8, w0, #0x1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 ; CHECK-NEXT: mov z2.s, w8 @@ -109,8 +109,8 @@ define <4 x float> @select_v4f32(<4 x float> %op1, <4 x float> %op2, i1 %mask) { define void @select_v8f32(ptr %a, ptr %b, i1 %mask) { ; CHECK-LABEL: select_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and w8, w2, #0x1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z0.s, w8 ; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0 ; CHECK-NEXT: ldr q0, [x0] @@ -150,9 +150,9 @@ define <1 x double> @select_v1f64(<1 x double> %op1, <1 x double> %op2, i1 %mask define <2 x double> @select_v2f64(<2 x double> %op1, <2 x double> %op2, i1 %mask) { ; CHECK-LABEL: select_v2f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 ; CHECK-NEXT: and x8, x0, #0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 ; CHECK-NEXT: mov z2.d, x8 @@ -167,9 +167,9 @@ define <2 x double> @select_v2f64(<2 x double> %op1, <2 x double> %op2, i1 %mask define void @select_v4f64(ptr %a, ptr %b, i1 %mask) { ; CHECK-LABEL: select_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $w2 killed $w2 def $x2 ; CHECK-NEXT: and x8, x2, #0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z0.d, x8 ; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 ; CHECK-NEXT: ldr q0, [x0] diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-to-int.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-to-int.ll index d6adf9cf0ad6..58eae212d799 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-to-int.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-to-int.ll @@ -36,8 +36,8 @@ define void @fcvtzu_v8f16_v8i16(ptr %a, ptr %b) { define void @fcvtzu_v16f16_v16i16(ptr %a, ptr %b) { ; CHECK-LABEL: fcvtzu_v16f16_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: fcvtzu z0.h, p0/m, z0.h ; CHECK-NEXT: fcvtzu z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x1] @@ -55,8 +55,8 @@ define void @fcvtzu_v16f16_v16i16(ptr %a, ptr %b) { define <2 x i32> @fcvtzu_v2f16_v2i32(<2 x half> %op1) { ; CHECK-LABEL: fcvtzu_v2f16_v2i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: fcvtzu z0.s, p0/m, z0.h ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 @@ -68,8 +68,8 @@ define <2 x i32> @fcvtzu_v2f16_v2i32(<2 x half> %op1) { define <4 x i32> @fcvtzu_v4f16_v4i32(<4 x half> %op1) { ; CHECK-LABEL: fcvtzu_v4f16_v4i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: fcvtzu z0.s, p0/m, z0.h ; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 @@ -302,8 +302,8 @@ define <4 x i16> @fcvtzu_v4f32_v4i16(<4 x float> %op1) { define <8 x i16> @fcvtzu_v8f32_v8i16(ptr %a) { ; CHECK-LABEL: fcvtzu_v8f32_v8i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: fcvtzu z1.s, p0/m, z1.s ; CHECK-NEXT: fcvtzu z0.s, p0/m, z0.s ; CHECK-NEXT: ptrue p0.h, vl4 @@ -320,8 +320,8 @@ define <8 x i16> @fcvtzu_v8f32_v8i16(ptr %a) { define void @fcvtzu_v16f32_v16i16(ptr %a, ptr %b) { ; CHECK-LABEL: fcvtzu_v16f32_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0, #32] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: fcvtzu z1.s, p0/m, z1.s ; CHECK-NEXT: fcvtzu z0.s, p0/m, z0.s @@ -373,8 +373,8 @@ define <4 x i32> @fcvtzu_v4f32_v4i32(<4 x float> %op1) { define void @fcvtzu_v8f32_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: fcvtzu_v8f32_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: fcvtzu z0.s, p0/m, z0.s ; CHECK-NEXT: fcvtzu z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x1] @@ -392,8 +392,8 @@ define void @fcvtzu_v8f32_v8i32(ptr %a, ptr %b) { define <1 x i64> @fcvtzu_v1f32_v1i64(<1 x float> %op1) { ; CHECK-LABEL: fcvtzu_v1f32_v1i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: fcvtzu z0.d, p0/m, z0.s ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 @@ -405,8 +405,8 @@ define <1 x i64> @fcvtzu_v1f32_v1i64(<1 x float> %op1) { define <2 x i64> @fcvtzu_v2f32_v2i64(<2 x float> %op1) { ; CHECK-LABEL: fcvtzu_v2f32_v2i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: fcvtzu z0.d, p0/m, z0.s ; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 @@ -491,8 +491,8 @@ define <4 x i16> @fcvtzu_v4f64_v4i16(ptr %a) { ; CHECK: // %bb.0: ; CHECK-NEXT: sub sp, sp, #16 ; CHECK-NEXT: .cfi_def_cfa_offset 16 -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.d ; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.d ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s @@ -520,8 +520,8 @@ define <8 x i16> @fcvtzu_v8f64_v8i16(ptr %a) { ; CHECK: // %bb.0: ; CHECK-NEXT: sub sp, sp, #16 ; CHECK-NEXT: .cfi_def_cfa_offset 16 -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0, #32] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q3, q2, [x0] ; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.d ; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.d @@ -563,24 +563,22 @@ define void @fcvtzu_v16f64_v16i16(ptr %a, ptr %b) { ; CHECK: // %bb.0: ; CHECK-NEXT: sub sp, sp, #32 ; CHECK-NEXT: .cfi_def_cfa_offset 32 -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0, #32] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q3, q2, [x0] -; CHECK-NEXT: ldr q6, [x0, #112] -; CHECK-NEXT: ldp q4, q5, [x0, #80] -; CHECK-NEXT: ldr q7, [x0, #64] +; CHECK-NEXT: ldp q4, q5, [x0, #96] ; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.d -; CHECK-NEXT: fcvtzs z2.d, p0/m, z2.d ; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.d +; CHECK-NEXT: fcvtzs z2.d, p0/m, z2.d +; CHECK-NEXT: ldp q6, q7, [x0, #64] ; CHECK-NEXT: fcvtzs z3.d, p0/m, z3.d -; CHECK-NEXT: fcvtzs z6.d, p0/m, z6.d ; CHECK-NEXT: fcvtzs z5.d, p0/m, z5.d ; CHECK-NEXT: fcvtzs z4.d, p0/m, z4.d ; CHECK-NEXT: uzp1 z1.s, z1.s, z1.s -; CHECK-NEXT: uzp1 z2.s, z2.s, z2.s ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s +; CHECK-NEXT: fcvtzs z6.d, p0/m, z6.d +; CHECK-NEXT: uzp1 z2.s, z2.s, z2.s ; CHECK-NEXT: uzp1 z3.s, z3.s, z3.s -; CHECK-NEXT: uzp1 z6.s, z6.s, z6.s ; CHECK-NEXT: uzp1 z5.s, z5.s, z5.s ; CHECK-NEXT: fmov w8, s1 ; CHECK-NEXT: mov z16.s, z1.s[1] @@ -606,25 +604,26 @@ define void @fcvtzu_v16f64_v16i16(ptr %a, ptr %b) { ; CHECK-NEXT: mov z3.s, z5.s[1] ; CHECK-NEXT: strh w8, [sp, #6] ; CHECK-NEXT: fmov w8, s2 -; CHECK-NEXT: mov z2.s, z6.s[1] +; CHECK-NEXT: uzp1 z2.s, z6.s, z6.s ; CHECK-NEXT: strh w8, [sp, #2] -; CHECK-NEXT: fmov w8, s6 -; CHECK-NEXT: strh w8, [sp, #28] ; CHECK-NEXT: fmov w8, s5 -; CHECK-NEXT: strh w8, [sp, #24] +; CHECK-NEXT: strh w8, [sp, #28] ; CHECK-NEXT: fmov w8, s1 ; CHECK-NEXT: mov z1.s, z1.s[1] -; CHECK-NEXT: strh w8, [sp, #20] +; CHECK-NEXT: strh w8, [sp, #24] ; CHECK-NEXT: fmov w8, s0 ; CHECK-NEXT: mov z0.s, z0.s[1] -; CHECK-NEXT: strh w8, [sp, #16] +; CHECK-NEXT: strh w8, [sp, #20] ; CHECK-NEXT: fmov w8, s2 -; CHECK-NEXT: strh w8, [sp, #30] +; CHECK-NEXT: mov z2.s, z2.s[1] +; CHECK-NEXT: strh w8, [sp, #16] ; CHECK-NEXT: fmov w8, s3 -; CHECK-NEXT: strh w8, [sp, #26] +; CHECK-NEXT: strh w8, [sp, #30] ; CHECK-NEXT: fmov w8, s1 -; CHECK-NEXT: strh w8, [sp, #22] +; CHECK-NEXT: strh w8, [sp, #26] ; CHECK-NEXT: fmov w8, s0 +; CHECK-NEXT: strh w8, [sp, #22] +; CHECK-NEXT: fmov w8, s2 ; CHECK-NEXT: strh w8, [sp, #18] ; CHECK-NEXT: ldp q1, q0, [sp] ; CHECK-NEXT: stp q1, q0, [x1] @@ -669,8 +668,8 @@ define <2 x i32> @fcvtzu_v2f64_v2i32(<2 x double> %op1) { define <4 x i32> @fcvtzu_v4f64_v4i32(ptr %a) { ; CHECK-LABEL: fcvtzu_v4f64_v4i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: fcvtzu z1.d, p0/m, z1.d ; CHECK-NEXT: fcvtzu z0.d, p0/m, z0.d ; CHECK-NEXT: ptrue p0.s, vl2 @@ -687,8 +686,8 @@ define <4 x i32> @fcvtzu_v4f64_v4i32(ptr %a) { define void @fcvtzu_v8f64_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: fcvtzu_v8f64_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0, #32] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: fcvtzu z1.d, p0/m, z1.d ; CHECK-NEXT: fcvtzu z0.d, p0/m, z0.d @@ -740,8 +739,8 @@ define <2 x i64> @fcvtzu_v2f64_v2i64(<2 x double> %op1) { define void @fcvtzu_v4f64_v4i64(ptr %a, ptr %b) { ; CHECK-LABEL: fcvtzu_v4f64_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: fcvtzu z0.d, p0/m, z0.d ; CHECK-NEXT: fcvtzu z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x1] @@ -785,8 +784,8 @@ define void @fcvtzs_v8f16_v8i16(ptr %a, ptr %b) { define void @fcvtzs_v16f16_v16i16(ptr %a, ptr %b) { ; CHECK-LABEL: fcvtzs_v16f16_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: fcvtzs z0.h, p0/m, z0.h ; CHECK-NEXT: fcvtzs z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x1] @@ -804,8 +803,8 @@ define void @fcvtzs_v16f16_v16i16(ptr %a, ptr %b) { define <2 x i32> @fcvtzs_v2f16_v2i32(<2 x half> %op1) { ; CHECK-LABEL: fcvtzs_v2f16_v2i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: fcvtzs z0.s, p0/m, z0.h ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 @@ -817,8 +816,8 @@ define <2 x i32> @fcvtzs_v2f16_v2i32(<2 x half> %op1) { define <4 x i32> @fcvtzs_v4f16_v4i32(<4 x half> %op1) { ; CHECK-LABEL: fcvtzs_v4f16_v4i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: fcvtzs z0.s, p0/m, z0.h ; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 @@ -1052,8 +1051,8 @@ define <4 x i16> @fcvtzs_v4f32_v4i16(<4 x float> %op1) { define <8 x i16> @fcvtzs_v8f32_v8i16(ptr %a) { ; CHECK-LABEL: fcvtzs_v8f32_v8i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: fcvtzs z1.s, p0/m, z1.s ; CHECK-NEXT: fcvtzs z0.s, p0/m, z0.s ; CHECK-NEXT: ptrue p0.h, vl4 @@ -1070,8 +1069,8 @@ define <8 x i16> @fcvtzs_v8f32_v8i16(ptr %a) { define void @fcvtzs_v16f32_v16i16(ptr %a, ptr %b) { ; CHECK-LABEL: fcvtzs_v16f32_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0, #32] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: fcvtzs z1.s, p0/m, z1.s ; CHECK-NEXT: fcvtzs z0.s, p0/m, z0.s @@ -1123,8 +1122,8 @@ define <4 x i32> @fcvtzs_v4f32_v4i32(<4 x float> %op1) { define void @fcvtzs_v8f32_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: fcvtzs_v8f32_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: fcvtzs z0.s, p0/m, z0.s ; CHECK-NEXT: fcvtzs z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x1] @@ -1142,8 +1141,8 @@ define void @fcvtzs_v8f32_v8i32(ptr %a, ptr %b) { define <1 x i64> @fcvtzs_v1f32_v1i64(<1 x float> %op1) { ; CHECK-LABEL: fcvtzs_v1f32_v1i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.s ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 @@ -1155,8 +1154,8 @@ define <1 x i64> @fcvtzs_v1f32_v1i64(<1 x float> %op1) { define <2 x i64> @fcvtzs_v2f32_v2i64(<2 x float> %op1) { ; CHECK-LABEL: fcvtzs_v2f32_v2i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.s ; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 @@ -1243,8 +1242,8 @@ define <4 x i16> @fcvtzs_v4f64_v4i16(ptr %a) { ; CHECK: // %bb.0: ; CHECK-NEXT: sub sp, sp, #16 ; CHECK-NEXT: .cfi_def_cfa_offset 16 -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.d ; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.d ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s @@ -1272,8 +1271,8 @@ define <8 x i16> @fcvtzs_v8f64_v8i16(ptr %a) { ; CHECK: // %bb.0: ; CHECK-NEXT: sub sp, sp, #16 ; CHECK-NEXT: .cfi_def_cfa_offset 16 -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0, #32] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q3, q2, [x0] ; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.d ; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.d @@ -1315,24 +1314,22 @@ define void @fcvtzs_v16f64_v16i16(ptr %a, ptr %b) { ; CHECK: // %bb.0: ; CHECK-NEXT: sub sp, sp, #32 ; CHECK-NEXT: .cfi_def_cfa_offset 32 -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0, #32] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q3, q2, [x0] -; CHECK-NEXT: ldr q6, [x0, #112] -; CHECK-NEXT: ldp q4, q5, [x0, #80] -; CHECK-NEXT: ldr q7, [x0, #64] +; CHECK-NEXT: ldp q4, q5, [x0, #96] ; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.d -; CHECK-NEXT: fcvtzs z2.d, p0/m, z2.d ; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.d +; CHECK-NEXT: fcvtzs z2.d, p0/m, z2.d +; CHECK-NEXT: ldp q6, q7, [x0, #64] ; CHECK-NEXT: fcvtzs z3.d, p0/m, z3.d -; CHECK-NEXT: fcvtzs z6.d, p0/m, z6.d ; CHECK-NEXT: fcvtzs z5.d, p0/m, z5.d ; CHECK-NEXT: fcvtzs z4.d, p0/m, z4.d ; CHECK-NEXT: uzp1 z1.s, z1.s, z1.s -; CHECK-NEXT: uzp1 z2.s, z2.s, z2.s ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s +; CHECK-NEXT: fcvtzs z6.d, p0/m, z6.d +; CHECK-NEXT: uzp1 z2.s, z2.s, z2.s ; CHECK-NEXT: uzp1 z3.s, z3.s, z3.s -; CHECK-NEXT: uzp1 z6.s, z6.s, z6.s ; CHECK-NEXT: uzp1 z5.s, z5.s, z5.s ; CHECK-NEXT: fmov w8, s1 ; CHECK-NEXT: mov z16.s, z1.s[1] @@ -1358,25 +1355,26 @@ define void @fcvtzs_v16f64_v16i16(ptr %a, ptr %b) { ; CHECK-NEXT: mov z3.s, z5.s[1] ; CHECK-NEXT: strh w8, [sp, #6] ; CHECK-NEXT: fmov w8, s2 -; CHECK-NEXT: mov z2.s, z6.s[1] +; CHECK-NEXT: uzp1 z2.s, z6.s, z6.s ; CHECK-NEXT: strh w8, [sp, #2] -; CHECK-NEXT: fmov w8, s6 -; CHECK-NEXT: strh w8, [sp, #28] ; CHECK-NEXT: fmov w8, s5 -; CHECK-NEXT: strh w8, [sp, #24] +; CHECK-NEXT: strh w8, [sp, #28] ; CHECK-NEXT: fmov w8, s1 ; CHECK-NEXT: mov z1.s, z1.s[1] -; CHECK-NEXT: strh w8, [sp, #20] +; CHECK-NEXT: strh w8, [sp, #24] ; CHECK-NEXT: fmov w8, s0 ; CHECK-NEXT: mov z0.s, z0.s[1] -; CHECK-NEXT: strh w8, [sp, #16] +; CHECK-NEXT: strh w8, [sp, #20] ; CHECK-NEXT: fmov w8, s2 -; CHECK-NEXT: strh w8, [sp, #30] +; CHECK-NEXT: mov z2.s, z2.s[1] +; CHECK-NEXT: strh w8, [sp, #16] ; CHECK-NEXT: fmov w8, s3 -; CHECK-NEXT: strh w8, [sp, #26] +; CHECK-NEXT: strh w8, [sp, #30] ; CHECK-NEXT: fmov w8, s1 -; CHECK-NEXT: strh w8, [sp, #22] +; CHECK-NEXT: strh w8, [sp, #26] ; CHECK-NEXT: fmov w8, s0 +; CHECK-NEXT: strh w8, [sp, #22] +; CHECK-NEXT: fmov w8, s2 ; CHECK-NEXT: strh w8, [sp, #18] ; CHECK-NEXT: ldp q1, q0, [sp] ; CHECK-NEXT: stp q1, q0, [x1] @@ -1421,8 +1419,8 @@ define <2 x i32> @fcvtzs_v2f64_v2i32(<2 x double> %op1) { define <4 x i32> @fcvtzs_v4f64_v4i32(ptr %a) { ; CHECK-LABEL: fcvtzs_v4f64_v4i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.d ; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.d ; CHECK-NEXT: ptrue p0.s, vl2 @@ -1439,8 +1437,8 @@ define <4 x i32> @fcvtzs_v4f64_v4i32(ptr %a) { define void @fcvtzs_v8f64_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: fcvtzs_v8f64_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0, #32] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.d ; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.d @@ -1492,8 +1490,8 @@ define <2 x i64> @fcvtzs_v2f64_v2i64(<2 x double> %op1) { define void @fcvtzs_v4f64_v4i64(ptr %a, ptr %b) { ; CHECK-LABEL: fcvtzs_v4f64_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.d ; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x1] diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-vselect.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-vselect.ll index ee8704284def..4c5a6fe2fd23 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-vselect.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-vselect.ll @@ -71,8 +71,8 @@ define <8 x half> @select_v8f16(<8 x half> %op1, <8 x half> %op2, <8 x i1> %mask define void @select_v16f16(ptr %a, ptr %b) { ; CHECK-LABEL: select_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q2, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q3, [x1] ; CHECK-NEXT: fcmeq p1.h, p0/z, z0.h, z1.h ; CHECK-NEXT: fcmeq p0.h, p0/z, z2.h, z3.h @@ -128,8 +128,8 @@ define <4 x float> @select_v4f32(<4 x float> %op1, <4 x float> %op2, <4 x i1> %m define void @select_v8f32(ptr %a, ptr %b) { ; CHECK-LABEL: select_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q2, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q3, [x1] ; CHECK-NEXT: fcmeq p1.s, p0/z, z0.s, z1.s ; CHECK-NEXT: fcmeq p0.s, p0/z, z2.s, z3.s @@ -186,8 +186,8 @@ define <2 x double> @select_v2f64(<2 x double> %op1, <2 x double> %op2, <2 x i1> define void @select_v4f64(ptr %a, ptr %b) { ; CHECK-LABEL: select_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q2, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q3, [x1] ; CHECK-NEXT: fcmeq p1.d, p0/z, z0.d, z1.d ; CHECK-NEXT: fcmeq p0.d, p0/z, z2.d, z3.d diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-insert-vector-elt.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-insert-vector-elt.ll index 0b3e7695e6a0..4aa965777c74 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-insert-vector-elt.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-insert-vector-elt.ll @@ -11,9 +11,9 @@ target triple = "aarch64-unknown-linux-gnu" define <4 x i8> @insertelement_v4i8(<4 x i8> %op1) { ; CHECK-LABEL: insertelement_v4i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w8, #3 // =0x3 ; CHECK-NEXT: index z1.h, #0, #1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w8 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: mov w8, #5 // =0x5 @@ -28,9 +28,9 @@ define <4 x i8> @insertelement_v4i8(<4 x i8> %op1) { define <8 x i8> @insertelement_v8i8(<8 x i8> %op1) { ; CHECK-LABEL: insertelement_v8i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov w8, #7 // =0x7 ; CHECK-NEXT: index z1.b, #0, #1 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z2.b, w8 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: mov w8, #5 // =0x5 @@ -45,9 +45,9 @@ define <8 x i8> @insertelement_v8i8(<8 x i8> %op1) { define <16 x i8> @insertelement_v16i8(<16 x i8> %op1) { ; CHECK-LABEL: insertelement_v16i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov w8, #15 // =0xf ; CHECK-NEXT: index z1.b, #0, #1 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z2.b, w8 ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: mov w8, #5 // =0x5 @@ -62,9 +62,9 @@ define <16 x i8> @insertelement_v16i8(<16 x i8> %op1) { define <32 x i8> @insertelement_v32i8(<32 x i8> %op1) { ; CHECK-LABEL: insertelement_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov w8, #15 // =0xf ; CHECK-NEXT: index z2.b, #0, #1 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z3.b, w8 ; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 ; CHECK-NEXT: mov w8, #5 // =0x5 @@ -80,9 +80,9 @@ define <32 x i8> @insertelement_v32i8(<32 x i8> %op1) { define <2 x i16> @insertelement_v2i16(<2 x i16> %op1) { ; CHECK-LABEL: insertelement_v2i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #1 // =0x1 ; CHECK-NEXT: index z1.s, #0, #1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, w8 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: mov w8, #5 // =0x5 @@ -97,9 +97,9 @@ define <2 x i16> @insertelement_v2i16(<2 x i16> %op1) { define <4 x i16> @insertelement_v4i16(<4 x i16> %op1) { ; CHECK-LABEL: insertelement_v4i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w8, #3 // =0x3 ; CHECK-NEXT: index z1.h, #0, #1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w8 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: mov w8, #5 // =0x5 @@ -114,9 +114,9 @@ define <4 x i16> @insertelement_v4i16(<4 x i16> %op1) { define <8 x i16> @insertelement_v8i16(<8 x i16> %op1) { ; CHECK-LABEL: insertelement_v8i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w8, #7 // =0x7 ; CHECK-NEXT: index z1.h, #0, #1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w8 ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: mov w8, #5 // =0x5 @@ -131,9 +131,9 @@ define <8 x i16> @insertelement_v8i16(<8 x i16> %op1) { define <16 x i16> @insertelement_v16i16(<16 x i16> %op1) { ; CHECK-LABEL: insertelement_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w8, #7 // =0x7 ; CHECK-NEXT: index z2.h, #0, #1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z3.h, w8 ; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 ; CHECK-NEXT: mov w8, #5 // =0x5 @@ -149,9 +149,9 @@ define <16 x i16> @insertelement_v16i16(<16 x i16> %op1) { define <2 x i32> @insertelement_v2i32(<2 x i32> %op1) { ; CHECK-LABEL: insertelement_v2i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #1 // =0x1 ; CHECK-NEXT: index z1.s, #0, #1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, w8 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: mov w8, #5 // =0x5 @@ -166,9 +166,9 @@ define <2 x i32> @insertelement_v2i32(<2 x i32> %op1) { define <4 x i32> @insertelement_v4i32(<4 x i32> %op1) { ; CHECK-LABEL: insertelement_v4i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #3 // =0x3 ; CHECK-NEXT: index z1.s, #0, #1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, w8 ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: mov w8, #5 // =0x5 @@ -183,9 +183,9 @@ define <4 x i32> @insertelement_v4i32(<4 x i32> %op1) { define <8 x i32> @insertelement_v8i32(ptr %a) { ; CHECK-LABEL: insertelement_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #3 // =0x3 ; CHECK-NEXT: index z0.s, #0, #1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: mov w8, #5 // =0x5 ; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, z1.s @@ -212,9 +212,9 @@ define <1 x i64> @insertelement_v1i64(<1 x i64> %op1) { define <2 x i64> @insertelement_v2i64(<2 x i64> %op1) { ; CHECK-LABEL: insertelement_v2i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, #1 // =0x1 ; CHECK-NEXT: index z1.d, #0, #1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z2.d, x8 ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: mov w8, #5 // =0x5 @@ -229,9 +229,9 @@ define <2 x i64> @insertelement_v2i64(<2 x i64> %op1) { define <4 x i64> @insertelement_v4i64(ptr %a) { ; CHECK-LABEL: insertelement_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, #1 // =0x1 ; CHECK-NEXT: index z0.d, #0, #1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: mov w8, #5 // =0x5 ; CHECK-NEXT: cmpeq p0.d, p0/z, z0.d, z1.d @@ -264,9 +264,9 @@ define <2 x half> @insertelement_v2f16(<2 x half> %op1) { define <4 x half> @insertelement_v4f16(<4 x half> %op1) { ; CHECK-LABEL: insertelement_v4f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w8, #3 // =0x3 ; CHECK-NEXT: index z1.h, #0, #1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w8 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: cmpeq p0.h, p0/z, z1.h, z2.h @@ -281,9 +281,9 @@ define <4 x half> @insertelement_v4f16(<4 x half> %op1) { define <8 x half> @insertelement_v8f16(<8 x half> %op1) { ; CHECK-LABEL: insertelement_v8f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w8, #7 // =0x7 ; CHECK-NEXT: index z1.h, #0, #1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w8 ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: cmpeq p0.h, p0/z, z1.h, z2.h @@ -298,9 +298,9 @@ define <8 x half> @insertelement_v8f16(<8 x half> %op1) { define <16 x half> @insertelement_v16f16(ptr %a) { ; CHECK-LABEL: insertelement_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w8, #7 // =0x7 ; CHECK-NEXT: index z0.h, #0, #1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: fmov h2, #5.00000000 ; CHECK-NEXT: cmpeq p0.h, p0/z, z0.h, z1.h @@ -317,9 +317,9 @@ define <16 x half> @insertelement_v16f16(ptr %a) { define <2 x float> @insertelement_v2f32(<2 x float> %op1) { ; CHECK-LABEL: insertelement_v2f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #1 // =0x1 ; CHECK-NEXT: index z1.s, #0, #1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, w8 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: cmpeq p0.s, p0/z, z1.s, z2.s @@ -334,9 +334,9 @@ define <2 x float> @insertelement_v2f32(<2 x float> %op1) { define <4 x float> @insertelement_v4f32(<4 x float> %op1) { ; CHECK-LABEL: insertelement_v4f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #3 // =0x3 ; CHECK-NEXT: index z1.s, #0, #1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, w8 ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: cmpeq p0.s, p0/z, z1.s, z2.s @@ -351,9 +351,9 @@ define <4 x float> @insertelement_v4f32(<4 x float> %op1) { define <8 x float> @insertelement_v8f32(ptr %a) { ; CHECK-LABEL: insertelement_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #3 // =0x3 ; CHECK-NEXT: index z0.s, #0, #1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: fmov s2, #5.00000000 ; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, z1.s @@ -379,9 +379,9 @@ define <1 x double> @insertelement_v1f64(<1 x double> %op1) { define <2 x double> @insertelement_v2f64(<2 x double> %op1) { ; CHECK-LABEL: insertelement_v2f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, #1 // =0x1 ; CHECK-NEXT: index z1.d, #0, #1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z2.d, x8 ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: cmpeq p0.d, p0/z, z1.d, z2.d @@ -396,9 +396,9 @@ define <2 x double> @insertelement_v2f64(<2 x double> %op1) { define <4 x double> @insertelement_v4f64(ptr %a) { ; CHECK-LABEL: insertelement_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, #1 // =0x1 ; CHECK-NEXT: index z0.d, #0, #1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: fmov d2, #5.00000000 ; CHECK-NEXT: cmpeq p0.d, p0/z, z0.d, z1.d diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-arith.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-arith.ll index e3c4b6f1cb53..8baa87c6d686 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-arith.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-arith.ll @@ -262,8 +262,8 @@ define <16 x i8> @mul_v16i8(<16 x i8> %op1, <16 x i8> %op2) { define void @mul_v32i8(ptr %a, ptr %b) { ; SVE-LABEL: mul_v32i8: ; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.b, vl16 ; SVE-NEXT: ldp q0, q3, [x1] +; SVE-NEXT: ptrue p0.b, vl16 ; SVE-NEXT: ldp q1, q2, [x0] ; SVE-NEXT: mul z0.b, p0/m, z0.b, z1.b ; SVE-NEXT: movprfx z1, z2 @@ -352,8 +352,8 @@ define <8 x i16> @mul_v8i16(<8 x i16> %op1, <8 x i16> %op2) { define void @mul_v16i16(ptr %a, ptr %b) { ; SVE-LABEL: mul_v16i16: ; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.h, vl8 ; SVE-NEXT: ldp q0, q3, [x1] +; SVE-NEXT: ptrue p0.h, vl8 ; SVE-NEXT: ldp q1, q2, [x0] ; SVE-NEXT: mul z0.h, p0/m, z0.h, z1.h ; SVE-NEXT: movprfx z1, z2 @@ -421,8 +421,8 @@ define <4 x i32> @mul_v4i32(<4 x i32> %op1, <4 x i32> %op2) { define void @mul_v8i32(ptr %a, ptr %b) { ; SVE-LABEL: mul_v8i32: ; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.s, vl4 ; SVE-NEXT: ldp q0, q3, [x1] +; SVE-NEXT: ptrue p0.s, vl4 ; SVE-NEXT: ldp q1, q2, [x0] ; SVE-NEXT: mul z0.s, p0/m, z0.s, z1.s ; SVE-NEXT: movprfx z1, z2 @@ -490,8 +490,8 @@ define <2 x i64> @mul_v2i64(<2 x i64> %op1, <2 x i64> %op2) { define void @mul_v4i64(ptr %a, ptr %b) { ; SVE-LABEL: mul_v4i64: ; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.d, vl2 ; SVE-NEXT: ldp q0, q3, [x1] +; SVE-NEXT: ptrue p0.d, vl2 ; SVE-NEXT: ldp q1, q2, [x0] ; SVE-NEXT: mul z0.d, p0/m, z0.d, z1.d ; SVE-NEXT: movprfx z1, z2 @@ -746,8 +746,8 @@ define <16 x i8> @abs_v16i8(<16 x i8> %op1) { define void @abs_v32i8(ptr %a) { ; CHECK-LABEL: abs_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: abs z0.b, p0/m, z0.b ; CHECK-NEXT: abs z1.b, p0/m, z1.b ; CHECK-NEXT: stp q0, q1, [x0] @@ -798,8 +798,8 @@ define <8 x i16> @abs_v8i16(<8 x i16> %op1) { define void @abs_v16i16(ptr %a) { ; CHECK-LABEL: abs_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: abs z0.h, p0/m, z0.h ; CHECK-NEXT: abs z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x0] @@ -837,8 +837,8 @@ define <4 x i32> @abs_v4i32(<4 x i32> %op1) { define void @abs_v8i32(ptr %a) { ; CHECK-LABEL: abs_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: abs z0.s, p0/m, z0.s ; CHECK-NEXT: abs z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -876,8 +876,8 @@ define <2 x i64> @abs_v2i64(<2 x i64> %op1) { define void @abs_v4i64(ptr %a) { ; CHECK-LABEL: abs_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: abs z0.d, p0/m, z0.d ; CHECK-NEXT: abs z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-compares.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-compares.ll index 6200e44218a9..73c1eac99dd3 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-compares.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-compares.ll @@ -41,8 +41,8 @@ define <16 x i8> @icmp_eq_v16i8(<16 x i8> %op1, <16 x i8> %op2) { define void @icmp_eq_v32i8(ptr %a, ptr %b) { ; CHECK-LABEL: icmp_eq_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: cmpeq p1.b, p0/z, z1.b, z0.b ; CHECK-NEXT: cmpeq p0.b, p0/z, z2.b, z3.b @@ -91,8 +91,8 @@ define <8 x i16> @icmp_eq_v8i16(<8 x i16> %op1, <8 x i16> %op2) { define void @icmp_eq_v16i16(ptr %a, ptr %b) { ; CHECK-LABEL: icmp_eq_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: cmpeq p1.h, p0/z, z1.h, z0.h ; CHECK-NEXT: cmpeq p0.h, p0/z, z2.h, z3.h @@ -141,8 +141,8 @@ define <4 x i32> @icmp_eq_v4i32(<4 x i32> %op1, <4 x i32> %op2) { define void @icmp_eq_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: icmp_eq_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: cmpeq p1.s, p0/z, z1.s, z0.s ; CHECK-NEXT: cmpeq p0.s, p0/z, z2.s, z3.s @@ -191,8 +191,8 @@ define <2 x i64> @icmp_eq_v2i64(<2 x i64> %op1, <2 x i64> %op2) { define void @icmp_eq_v4i64(ptr %a, ptr %b) { ; CHECK-LABEL: icmp_eq_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: cmpeq p1.d, p0/z, z1.d, z0.d ; CHECK-NEXT: cmpeq p0.d, p0/z, z2.d, z3.d @@ -215,8 +215,8 @@ define void @icmp_eq_v4i64(ptr %a, ptr %b) { define void @icmp_ne_v32i8(ptr %a, ptr %b) { ; CHECK-LABEL: icmp_ne_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: cmpne p1.b, p0/z, z1.b, z0.b ; CHECK-NEXT: cmpne p0.b, p0/z, z2.b, z3.b @@ -261,8 +261,8 @@ define void @icmp_sge_v8i16(ptr %a, ptr %b) { define void @icmp_sgt_v16i16(ptr %a, ptr %b) { ; CHECK-LABEL: icmp_sgt_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: cmpgt p1.h, p0/z, z1.h, z0.h ; CHECK-NEXT: cmpgt p0.h, p0/z, z2.h, z3.h @@ -307,8 +307,8 @@ define void @icmp_sle_v4i32(ptr %a, ptr %b) { define void @icmp_slt_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: icmp_slt_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: cmpgt p1.s, p0/z, z0.s, z1.s ; CHECK-NEXT: cmpgt p0.s, p0/z, z3.s, z2.s diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-div.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-div.ll index fcf4f21c6ea8..5158dda37a8b 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-div.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-div.ll @@ -86,8 +86,8 @@ define <16 x i8> @sdiv_v16i8(<16 x i8> %op1, <16 x i8> %op2) { ; CHECK-NEXT: sdivr z3.s, p0/m, z3.s, z5.s ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h ; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z1.s -; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: uzp1 z1.h, z4.h, z4.h +; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h ; CHECK-NEXT: splice z1.h, p0, z1.h, z2.h ; CHECK-NEXT: uzp1 z1.b, z1.b, z1.b @@ -163,18 +163,18 @@ define void @sdiv_v32i8(ptr %a, ptr %b) { ; CHECK-NEXT: uzp1 z5.h, z5.h, z5.h ; CHECK-NEXT: sdiv z6.s, p0/m, z6.s, z7.s ; CHECK-NEXT: ptrue p0.h, vl4 -; CHECK-NEXT: uzp1 z7.h, z16.h, z16.h ; CHECK-NEXT: splice z4.h, p0, z4.h, z5.h ; CHECK-NEXT: splice z0.h, p0, z0.h, z1.h ; CHECK-NEXT: splice z2.h, p0, z2.h, z3.h +; CHECK-NEXT: uzp1 z7.h, z16.h, z16.h ; CHECK-NEXT: uzp1 z1.b, z4.b, z4.b ; CHECK-NEXT: uzp1 z0.b, z0.b, z0.b ; CHECK-NEXT: uzp1 z2.b, z2.b, z2.b ; CHECK-NEXT: uzp1 z6.h, z6.h, z6.h ; CHECK-NEXT: splice z7.h, p0, z7.h, z6.h ; CHECK-NEXT: ptrue p0.b, vl8 -; CHECK-NEXT: uzp1 z3.b, z7.b, z7.b ; CHECK-NEXT: splice z2.b, p0, z2.b, z0.b +; CHECK-NEXT: uzp1 z3.b, z7.b, z7.b ; CHECK-NEXT: splice z3.b, p0, z3.b, z1.b ; CHECK-NEXT: stp q3, q2, [x0] ; CHECK-NEXT: ret @@ -203,9 +203,9 @@ define <2 x i16> @sdiv_v2i16(<2 x i16> %op1, <2 x i16> %op2) { define <4 x i16> @sdiv_v4i16(<4 x i16> %op1, <4 x i16> %op2) { ; CHECK-LABEL: sdiv_v4i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: sunpklo z1.s, z1.h ; CHECK-NEXT: sunpklo z0.s, z0.h ; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z1.s @@ -272,8 +272,8 @@ define void @sdiv_v16i16(ptr %a, ptr %b) { ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h ; CHECK-NEXT: sdiv z3.s, p0/m, z3.s, z4.s ; CHECK-NEXT: ptrue p0.h, vl4 -; CHECK-NEXT: uzp1 z1.h, z5.h, z5.h ; CHECK-NEXT: splice z0.h, p0, z0.h, z2.h +; CHECK-NEXT: uzp1 z1.h, z5.h, z5.h ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h ; CHECK-NEXT: splice z3.h, p0, z3.h, z1.h ; CHECK-NEXT: stp q3, q0, [x0] @@ -314,8 +314,8 @@ define <4 x i32> @sdiv_v4i32(<4 x i32> %op1, <4 x i32> %op2) { define void @sdiv_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: sdiv_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: sdivr z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -358,8 +358,8 @@ define <2 x i64> @sdiv_v2i64(<2 x i64> %op1, <2 x i64> %op2) { define void @sdiv_v4i64(ptr %a, ptr %b) { ; CHECK-LABEL: sdiv_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: sdivr z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 @@ -453,8 +453,8 @@ define <16 x i8> @udiv_v16i8(<16 x i8> %op1, <16 x i8> %op2) { ; CHECK-NEXT: udivr z3.s, p0/m, z3.s, z5.s ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h ; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z1.s -; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: uzp1 z1.h, z4.h, z4.h +; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h ; CHECK-NEXT: splice z1.h, p0, z1.h, z2.h ; CHECK-NEXT: uzp1 z1.b, z1.b, z1.b @@ -530,18 +530,18 @@ define void @udiv_v32i8(ptr %a, ptr %b) { ; CHECK-NEXT: uzp1 z5.h, z5.h, z5.h ; CHECK-NEXT: udiv z6.s, p0/m, z6.s, z7.s ; CHECK-NEXT: ptrue p0.h, vl4 -; CHECK-NEXT: uzp1 z7.h, z16.h, z16.h ; CHECK-NEXT: splice z4.h, p0, z4.h, z5.h ; CHECK-NEXT: splice z0.h, p0, z0.h, z1.h ; CHECK-NEXT: splice z2.h, p0, z2.h, z3.h +; CHECK-NEXT: uzp1 z7.h, z16.h, z16.h ; CHECK-NEXT: uzp1 z1.b, z4.b, z4.b ; CHECK-NEXT: uzp1 z0.b, z0.b, z0.b ; CHECK-NEXT: uzp1 z2.b, z2.b, z2.b ; CHECK-NEXT: uzp1 z6.h, z6.h, z6.h ; CHECK-NEXT: splice z7.h, p0, z7.h, z6.h ; CHECK-NEXT: ptrue p0.b, vl8 -; CHECK-NEXT: uzp1 z3.b, z7.b, z7.b ; CHECK-NEXT: splice z2.b, p0, z2.b, z0.b +; CHECK-NEXT: uzp1 z3.b, z7.b, z7.b ; CHECK-NEXT: splice z3.b, p0, z3.b, z1.b ; CHECK-NEXT: stp q3, q2, [x0] ; CHECK-NEXT: ret @@ -555,9 +555,9 @@ define void @udiv_v32i8(ptr %a, ptr %b) { define <2 x i16> @udiv_v2i16(<2 x i16> %op1, <2 x i16> %op2) { ; CHECK-LABEL: udiv_v2i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: and z1.s, z1.s, #0xffff ; CHECK-NEXT: and z0.s, z0.s, #0xffff ; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z1.s @@ -570,9 +570,9 @@ define <2 x i16> @udiv_v2i16(<2 x i16> %op1, <2 x i16> %op2) { define <4 x i16> @udiv_v4i16(<4 x i16> %op1, <4 x i16> %op2) { ; CHECK-LABEL: udiv_v4i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: uunpklo z1.s, z1.h ; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z1.s @@ -639,8 +639,8 @@ define void @udiv_v16i16(ptr %a, ptr %b) { ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h ; CHECK-NEXT: udiv z3.s, p0/m, z3.s, z4.s ; CHECK-NEXT: ptrue p0.h, vl4 -; CHECK-NEXT: uzp1 z1.h, z5.h, z5.h ; CHECK-NEXT: splice z0.h, p0, z0.h, z2.h +; CHECK-NEXT: uzp1 z1.h, z5.h, z5.h ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h ; CHECK-NEXT: splice z3.h, p0, z3.h, z1.h ; CHECK-NEXT: stp q3, q0, [x0] @@ -681,8 +681,8 @@ define <4 x i32> @udiv_v4i32(<4 x i32> %op1, <4 x i32> %op2) { define void @udiv_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: udiv_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: udivr z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -725,8 +725,8 @@ define <2 x i64> @udiv_v2i64(<2 x i64> %op1, <2 x i64> %op2) { define void @udiv_v4i64(ptr %a, ptr %b) { ; CHECK-LABEL: udiv_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: udivr z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 @@ -743,10 +743,10 @@ define void @udiv_v4i64(ptr %a, ptr %b) { define void @udiv_constantsplat_v8i32(ptr %a) { ; SVE-LABEL: udiv_constantsplat_v8i32: ; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.s, vl4 ; SVE-NEXT: mov w8, #8969 // =0x2309 -; SVE-NEXT: movk w8, #22765, lsl #16 ; SVE-NEXT: ldp q1, q2, [x0] +; SVE-NEXT: movk w8, #22765, lsl #16 +; SVE-NEXT: ptrue p0.s, vl4 ; SVE-NEXT: mov z0.s, w8 ; SVE-NEXT: movprfx z3, z1 ; SVE-NEXT: umulh z3.s, p0/m, z3.s, z0.s diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-immediates.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-immediates.ll index 0785c67ce6f4..f028b3eeca25 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-immediates.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-immediates.ll @@ -221,8 +221,8 @@ define void @ashr_v4i64(ptr %a) { define void @icmp_eq_v32i8(ptr %a) { ; CHECK-LABEL: icmp_eq_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: cmpeq p1.b, p0/z, z0.b, #7 ; CHECK-NEXT: cmpeq p0.b, p0/z, z1.b, #7 ; CHECK-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff @@ -241,8 +241,8 @@ define void @icmp_eq_v32i8(ptr %a) { define void @icmp_sge_v16i16(ptr %a) { ; CHECK-LABEL: icmp_sge_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: cmpge p1.h, p0/z, z0.h, #15 ; CHECK-NEXT: cmpge p0.h, p0/z, z1.h, #15 ; CHECK-NEXT: mov z0.h, p1/z, #-1 // =0xffffffffffffffff @@ -261,8 +261,8 @@ define void @icmp_sge_v16i16(ptr %a) { define void @icmp_sgt_v8i32(ptr %a) { ; CHECK-LABEL: icmp_sgt_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: cmpgt p1.s, p0/z, z0.s, #-8 ; CHECK-NEXT: cmpgt p0.s, p0/z, z1.s, #-8 ; CHECK-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff @@ -281,8 +281,8 @@ define void @icmp_sgt_v8i32(ptr %a) { define void @icmp_ult_v4i64(ptr %a) { ; CHECK-LABEL: icmp_ult_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: cmplo p1.d, p0/z, z0.d, #63 ; CHECK-NEXT: cmplo p0.d, p0/z, z1.d, #63 ; CHECK-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-minmax.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-minmax.ll index d7600c6e6192..50cf9b73d9a7 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-minmax.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-minmax.ll @@ -37,8 +37,8 @@ define <16 x i8> @smax_v16i8(<16 x i8> %op1, <16 x i8> %op2) { define void @smax_v32i8(ptr %a, ptr %b) { ; CHECK-LABEL: smax_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: smax z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: movprfx z1, z2 @@ -81,8 +81,8 @@ define <8 x i16> @smax_v8i16(<8 x i16> %op1, <8 x i16> %op2) { define void @smax_v16i16(ptr %a, ptr %b) { ; CHECK-LABEL: smax_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: smax z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -125,8 +125,8 @@ define <4 x i32> @smax_v4i32(<4 x i32> %op1, <4 x i32> %op2) { define void @smax_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: smax_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: smax z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -171,8 +171,8 @@ define <2 x i64> @smax_v2i64(<2 x i64> %op1, <2 x i64> %op2) { define void @smax_v4i64(ptr %a, ptr %b) { ; CHECK-LABEL: smax_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: smax z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 @@ -219,8 +219,8 @@ define <16 x i8> @smin_v16i8(<16 x i8> %op1, <16 x i8> %op2) { define void @smin_v32i8(ptr %a, ptr %b) { ; CHECK-LABEL: smin_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: smin z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: movprfx z1, z2 @@ -263,8 +263,8 @@ define <8 x i16> @smin_v8i16(<8 x i16> %op1, <8 x i16> %op2) { define void @smin_v16i16(ptr %a, ptr %b) { ; CHECK-LABEL: smin_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: smin z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -307,8 +307,8 @@ define <4 x i32> @smin_v4i32(<4 x i32> %op1, <4 x i32> %op2) { define void @smin_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: smin_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: smin z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -353,8 +353,8 @@ define <2 x i64> @smin_v2i64(<2 x i64> %op1, <2 x i64> %op2) { define void @smin_v4i64(ptr %a, ptr %b) { ; CHECK-LABEL: smin_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: smin z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 @@ -401,8 +401,8 @@ define <16 x i8> @umax_v16i8(<16 x i8> %op1, <16 x i8> %op2) { define void @umax_v32i8(ptr %a, ptr %b) { ; CHECK-LABEL: umax_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: umax z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: movprfx z1, z2 @@ -445,8 +445,8 @@ define <8 x i16> @umax_v8i16(<8 x i16> %op1, <8 x i16> %op2) { define void @umax_v16i16(ptr %a, ptr %b) { ; CHECK-LABEL: umax_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: umax z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -489,8 +489,8 @@ define <4 x i32> @umax_v4i32(<4 x i32> %op1, <4 x i32> %op2) { define void @umax_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: umax_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: umax z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -535,8 +535,8 @@ define <2 x i64> @umax_v2i64(<2 x i64> %op1, <2 x i64> %op2) { define void @umax_v4i64(ptr %a, ptr %b) { ; CHECK-LABEL: umax_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: umax z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 @@ -583,8 +583,8 @@ define <16 x i8> @umin_v16i8(<16 x i8> %op1, <16 x i8> %op2) { define void @umin_v32i8(ptr %a, ptr %b) { ; CHECK-LABEL: umin_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: umin z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: movprfx z1, z2 @@ -627,8 +627,8 @@ define <8 x i16> @umin_v8i16(<8 x i16> %op1, <8 x i16> %op2) { define void @umin_v16i16(ptr %a, ptr %b) { ; CHECK-LABEL: umin_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: umin z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -671,8 +671,8 @@ define <4 x i32> @umin_v4i32(<4 x i32> %op1, <4 x i32> %op2) { define void @umin_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: umin_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: umin z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -717,8 +717,8 @@ define <2 x i64> @umin_v2i64(<2 x i64> %op1, <2 x i64> %op2) { define void @umin_v4i64(ptr %a, ptr %b) { ; CHECK-LABEL: umin_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: umin z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-mulh.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-mulh.ll index c48cb315a7aa..cb7fa53eac51 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-mulh.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-mulh.ll @@ -101,8 +101,8 @@ define <16 x i8> @smulh_v16i8(<16 x i8> %op1, <16 x i8> %op2) { define void @smulh_v32i8(ptr %a, ptr %b) { ; SVE-LABEL: smulh_v32i8: ; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.b, vl16 ; SVE-NEXT: ldp q0, q3, [x1] +; SVE-NEXT: ptrue p0.b, vl16 ; SVE-NEXT: ldp q1, q2, [x0] ; SVE-NEXT: smulh z0.b, p0/m, z0.b, z1.b ; SVE-NEXT: movprfx z1, z2 @@ -214,8 +214,8 @@ define <8 x i16> @smulh_v8i16(<8 x i16> %op1, <8 x i16> %op2) { define void @smulh_v16i16(ptr %a, ptr %b) { ; SVE-LABEL: smulh_v16i16: ; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.h, vl8 ; SVE-NEXT: ldp q0, q3, [x1] +; SVE-NEXT: ptrue p0.h, vl8 ; SVE-NEXT: ldp q1, q2, [x0] ; SVE-NEXT: smulh z0.h, p0/m, z0.h, z1.h ; SVE-NEXT: movprfx z1, z2 @@ -295,8 +295,8 @@ define <4 x i32> @smulh_v4i32(<4 x i32> %op1, <4 x i32> %op2) { define void @smulh_v8i32(ptr %a, ptr %b) { ; SVE-LABEL: smulh_v8i32: ; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.s, vl4 ; SVE-NEXT: ldp q0, q3, [x1] +; SVE-NEXT: ptrue p0.s, vl4 ; SVE-NEXT: ldp q1, q2, [x0] ; SVE-NEXT: smulh z0.s, p0/m, z0.s, z1.s ; SVE-NEXT: movprfx z1, z2 @@ -378,8 +378,8 @@ define <2 x i64> @smulh_v2i64(<2 x i64> %op1, <2 x i64> %op2) { define void @smulh_v4i64(ptr %a, ptr %b) { ; SVE-LABEL: smulh_v4i64: ; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.d, vl2 ; SVE-NEXT: ldp q0, q3, [x1] +; SVE-NEXT: ptrue p0.d, vl2 ; SVE-NEXT: ldp q1, q2, [x0] ; SVE-NEXT: smulh z0.d, p0/m, z0.d, z1.d ; SVE-NEXT: movprfx z1, z2 @@ -413,9 +413,9 @@ define void @smulh_v4i64(ptr %a, ptr %b) { define <4 x i8> @umulh_v4i8(<4 x i8> %op1, <4 x i8> %op2) { ; SVE-LABEL: umulh_v4i8: ; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.h, vl4 ; SVE-NEXT: // kill: def $d1 killed $d1 def $z1 ; SVE-NEXT: // kill: def $d0 killed $d0 def $z0 +; SVE-NEXT: ptrue p0.h, vl4 ; SVE-NEXT: and z0.h, z0.h, #0xff ; SVE-NEXT: and z1.h, z1.h, #0xff ; SVE-NEXT: mul z0.h, p0/m, z0.h, z1.h @@ -494,8 +494,8 @@ define <16 x i8> @umulh_v16i8(<16 x i8> %op1, <16 x i8> %op2) { define void @umulh_v32i8(ptr %a, ptr %b) { ; SVE-LABEL: umulh_v32i8: ; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.b, vl16 ; SVE-NEXT: ldp q0, q3, [x1] +; SVE-NEXT: ptrue p0.b, vl16 ; SVE-NEXT: ldp q1, q2, [x0] ; SVE-NEXT: umulh z0.b, p0/m, z0.b, z1.b ; SVE-NEXT: movprfx z1, z2 @@ -525,9 +525,9 @@ define void @umulh_v32i8(ptr %a, ptr %b) { define <2 x i16> @umulh_v2i16(<2 x i16> %op1, <2 x i16> %op2) { ; SVE-LABEL: umulh_v2i16: ; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.s, vl2 ; SVE-NEXT: // kill: def $d1 killed $d1 def $z1 ; SVE-NEXT: // kill: def $d0 killed $d0 def $z0 +; SVE-NEXT: ptrue p0.s, vl2 ; SVE-NEXT: and z0.s, z0.s, #0xffff ; SVE-NEXT: and z1.s, z1.s, #0xffff ; SVE-NEXT: mul z0.s, p0/m, z0.s, z1.s @@ -606,8 +606,8 @@ define <8 x i16> @umulh_v8i16(<8 x i16> %op1, <8 x i16> %op2) { define void @umulh_v16i16(ptr %a, ptr %b) { ; SVE-LABEL: umulh_v16i16: ; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.h, vl8 ; SVE-NEXT: ldp q0, q3, [x1] +; SVE-NEXT: ptrue p0.h, vl8 ; SVE-NEXT: ldp q1, q2, [x0] ; SVE-NEXT: umulh z0.h, p0/m, z0.h, z1.h ; SVE-NEXT: movprfx z1, z2 @@ -687,8 +687,8 @@ define <4 x i32> @umulh_v4i32(<4 x i32> %op1, <4 x i32> %op2) { define void @umulh_v8i32(ptr %a, ptr %b) { ; SVE-LABEL: umulh_v8i32: ; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.s, vl4 ; SVE-NEXT: ldp q0, q3, [x1] +; SVE-NEXT: ptrue p0.s, vl4 ; SVE-NEXT: ldp q1, q2, [x0] ; SVE-NEXT: umulh z0.s, p0/m, z0.s, z1.s ; SVE-NEXT: movprfx z1, z2 @@ -770,8 +770,8 @@ define <2 x i64> @umulh_v2i64(<2 x i64> %op1, <2 x i64> %op2) { define void @umulh_v4i64(ptr %a, ptr %b) { ; SVE-LABEL: umulh_v4i64: ; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.d, vl2 ; SVE-NEXT: ldp q0, q3, [x1] +; SVE-NEXT: ptrue p0.d, vl2 ; SVE-NEXT: ldp q1, q2, [x0] ; SVE-NEXT: umulh z0.d, p0/m, z0.d, z1.d ; SVE-NEXT: movprfx z1, z2 diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-reduce.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-reduce.ll index c51630ecd752..751f43768a51 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-reduce.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-reduce.ll @@ -37,8 +37,8 @@ define i8 @uaddv_v16i8(<16 x i8> %a) { define i8 @uaddv_v32i8(ptr %a) { ; CHECK-LABEL: uaddv_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: add z0.b, z1.b, z0.b ; CHECK-NEXT: uaddv d0, p0, z0.b ; CHECK-NEXT: fmov x0, d0 @@ -78,8 +78,8 @@ define i16 @uaddv_v8i16(<8 x i16> %a) { define i16 @uaddv_v16i16(ptr %a) { ; CHECK-LABEL: uaddv_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: add z0.h, z1.h, z0.h ; CHECK-NEXT: uaddv d0, p0, z0.h ; CHECK-NEXT: fmov x0, d0 @@ -119,8 +119,8 @@ define i32 @uaddv_v4i32(<4 x i32> %a) { define i32 @uaddv_v8i32(ptr %a) { ; CHECK-LABEL: uaddv_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: add z0.s, z1.s, z0.s ; CHECK-NEXT: uaddv d0, p0, z0.s ; CHECK-NEXT: fmov x0, d0 @@ -146,8 +146,8 @@ define i64 @uaddv_v2i64(<2 x i64> %a) { define i64 @uaddv_v4i64(ptr %a) { ; CHECK-LABEL: uaddv_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: add z0.d, z1.d, z0.d ; CHECK-NEXT: uaddv d0, p0, z0.d ; CHECK-NEXT: fmov x0, d0 @@ -188,8 +188,8 @@ define i8 @smaxv_v16i8(<16 x i8> %a) { define i8 @smaxv_v32i8(ptr %a) { ; CHECK-LABEL: smaxv_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: smax z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: smaxv b0, p0, z0.b ; CHECK-NEXT: fmov w0, s0 @@ -226,8 +226,8 @@ define i16 @smaxv_v8i16(<8 x i16> %a) { define i16 @smaxv_v16i16(ptr %a) { ; CHECK-LABEL: smaxv_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: smax z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: smaxv h0, p0, z0.h ; CHECK-NEXT: fmov w0, s0 @@ -264,8 +264,8 @@ define i32 @smaxv_v4i32(<4 x i32> %a) { define i32 @smaxv_v8i32(ptr %a) { ; CHECK-LABEL: smaxv_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: smax z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: smaxv s0, p0, z0.s ; CHECK-NEXT: fmov w0, s0 @@ -291,8 +291,8 @@ define i64 @smaxv_v2i64(<2 x i64> %a) { define i64 @smaxv_v4i64(ptr %a) { ; CHECK-LABEL: smaxv_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: smax z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: smaxv d0, p0, z0.d ; CHECK-NEXT: fmov x0, d0 @@ -333,8 +333,8 @@ define i8 @sminv_v16i8(<16 x i8> %a) { define i8 @sminv_v32i8(ptr %a) { ; CHECK-LABEL: sminv_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: smin z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: sminv b0, p0, z0.b ; CHECK-NEXT: fmov w0, s0 @@ -371,8 +371,8 @@ define i16 @sminv_v8i16(<8 x i16> %a) { define i16 @sminv_v16i16(ptr %a) { ; CHECK-LABEL: sminv_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: smin z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: sminv h0, p0, z0.h ; CHECK-NEXT: fmov w0, s0 @@ -409,8 +409,8 @@ define i32 @sminv_v4i32(<4 x i32> %a) { define i32 @sminv_v8i32(ptr %a) { ; CHECK-LABEL: sminv_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: smin z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: sminv s0, p0, z0.s ; CHECK-NEXT: fmov w0, s0 @@ -436,8 +436,8 @@ define i64 @sminv_v2i64(<2 x i64> %a) { define i64 @sminv_v4i64(ptr %a) { ; CHECK-LABEL: sminv_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: smin z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: sminv d0, p0, z0.d ; CHECK-NEXT: fmov x0, d0 @@ -478,8 +478,8 @@ define i8 @umaxv_v16i8(<16 x i8> %a) { define i8 @umaxv_v32i8(ptr %a) { ; CHECK-LABEL: umaxv_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: umax z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: umaxv b0, p0, z0.b ; CHECK-NEXT: fmov w0, s0 @@ -516,8 +516,8 @@ define i16 @umaxv_v8i16(<8 x i16> %a) { define i16 @umaxv_v16i16(ptr %a) { ; CHECK-LABEL: umaxv_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: umax z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: umaxv h0, p0, z0.h ; CHECK-NEXT: fmov w0, s0 @@ -554,8 +554,8 @@ define i32 @umaxv_v4i32(<4 x i32> %a) { define i32 @umaxv_v8i32(ptr %a) { ; CHECK-LABEL: umaxv_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: umax z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: umaxv s0, p0, z0.s ; CHECK-NEXT: fmov w0, s0 @@ -581,8 +581,8 @@ define i64 @umaxv_v2i64(<2 x i64> %a) { define i64 @umaxv_v4i64(ptr %a) { ; CHECK-LABEL: umaxv_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: umax z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: umaxv d0, p0, z0.d ; CHECK-NEXT: fmov x0, d0 @@ -623,8 +623,8 @@ define i8 @uminv_v16i8(<16 x i8> %a) { define i8 @uminv_v32i8(ptr %a) { ; CHECK-LABEL: uminv_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: umin z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: uminv b0, p0, z0.b ; CHECK-NEXT: fmov w0, s0 @@ -661,8 +661,8 @@ define i16 @uminv_v8i16(<8 x i16> %a) { define i16 @uminv_v16i16(ptr %a) { ; CHECK-LABEL: uminv_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: umin z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: uminv h0, p0, z0.h ; CHECK-NEXT: fmov w0, s0 @@ -699,8 +699,8 @@ define i32 @uminv_v4i32(<4 x i32> %a) { define i32 @uminv_v8i32(ptr %a) { ; CHECK-LABEL: uminv_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: umin z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: uminv s0, p0, z0.s ; CHECK-NEXT: fmov w0, s0 @@ -726,8 +726,8 @@ define i64 @uminv_v2i64(<2 x i64> %a) { define i64 @uminv_v4i64(ptr %a) { ; CHECK-LABEL: uminv_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: umin z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: uminv d0, p0, z0.d ; CHECK-NEXT: fmov x0, d0 diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-rem.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-rem.ll index 4a1209b942f4..d373a9063f85 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-rem.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-rem.ll @@ -65,7 +65,6 @@ define <16 x i8> @srem_v16i8(<16 x i8> %op1, <16 x i8> %op2) { ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: mov z3.d, z0.d ; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ptrue p1.b, vl16 ; CHECK-NEXT: ext z2.b, z2.b, z1.b, #8 ; CHECK-NEXT: ext z3.b, z3.b, z0.b, #8 ; CHECK-NEXT: sunpklo z2.h, z2.b @@ -91,15 +90,16 @@ define <16 x i8> @srem_v16i8(<16 x i8> %op1, <16 x i8> %op2) { ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h ; CHECK-NEXT: sdivr z3.s, p0/m, z3.s, z5.s ; CHECK-NEXT: ptrue p0.h, vl4 -; CHECK-NEXT: uzp1 z5.h, z6.h, z6.h ; CHECK-NEXT: splice z4.h, p0, z4.h, z2.h +; CHECK-NEXT: uzp1 z5.h, z6.h, z6.h ; CHECK-NEXT: uzp1 z2.b, z4.b, z4.b ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h ; CHECK-NEXT: splice z5.h, p0, z5.h, z3.h ; CHECK-NEXT: ptrue p0.b, vl8 ; CHECK-NEXT: uzp1 z3.b, z5.b, z5.b ; CHECK-NEXT: splice z3.b, p0, z3.b, z2.b -; CHECK-NEXT: mls z0.b, p1/m, z3.b, z1.b +; CHECK-NEXT: ptrue p0.b, vl16 +; CHECK-NEXT: mls z0.b, p0/m, z3.b, z1.b ; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 ; CHECK-NEXT: ret %res = srem <16 x i8> %op1, %op2 @@ -112,7 +112,6 @@ define void @srem_v32i8(ptr %a, ptr %b) { ; CHECK-NEXT: ldr q0, [x0, #16] ; CHECK-NEXT: ldr q1, [x1, #16] ; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ptrue p1.b, vl16 ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: mov z3.d, z0.d ; CHECK-NEXT: sunpklo z7.h, z1.b @@ -171,22 +170,23 @@ define void @srem_v32i8(ptr %a, ptr %b) { ; CHECK-NEXT: uzp1 z17.h, z17.h, z17.h ; CHECK-NEXT: sdivr z18.s, p0/m, z18.s, z20.s ; CHECK-NEXT: ptrue p0.h, vl4 -; CHECK-NEXT: uzp1 z19.h, z21.h, z21.h ; CHECK-NEXT: splice z16.h, p0, z16.h, z17.h ; CHECK-NEXT: splice z2.h, p0, z2.h, z5.h ; CHECK-NEXT: splice z6.h, p0, z6.h, z7.h +; CHECK-NEXT: uzp1 z19.h, z21.h, z21.h ; CHECK-NEXT: uzp1 z5.b, z16.b, z16.b ; CHECK-NEXT: uzp1 z2.b, z2.b, z2.b ; CHECK-NEXT: uzp1 z6.b, z6.b, z6.b ; CHECK-NEXT: uzp1 z18.h, z18.h, z18.h ; CHECK-NEXT: splice z19.h, p0, z19.h, z18.h ; CHECK-NEXT: ptrue p0.b, vl8 -; CHECK-NEXT: uzp1 z7.b, z19.b, z19.b ; CHECK-NEXT: splice z6.b, p0, z6.b, z2.b +; CHECK-NEXT: uzp1 z7.b, z19.b, z19.b ; CHECK-NEXT: splice z7.b, p0, z7.b, z5.b -; CHECK-NEXT: mls z0.b, p1/m, z6.b, z1.b +; CHECK-NEXT: ptrue p0.b, vl16 +; CHECK-NEXT: mls z0.b, p0/m, z6.b, z1.b ; CHECK-NEXT: movprfx z2, z3 -; CHECK-NEXT: mls z2.b, p1/m, z7.b, z4.b +; CHECK-NEXT: mls z2.b, p0/m, z7.b, z4.b ; CHECK-NEXT: stp q2, q0, [x0] ; CHECK-NEXT: ret %op1 = load <32 x i8>, ptr %a @@ -199,11 +199,11 @@ define void @srem_v32i8(ptr %a, ptr %b) { define <4 x i16> @srem_v4i16(<4 x i16> %op1, <4 x i16> %op2) { ; CHECK-LABEL: srem_v4i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: sunpklo z2.s, z1.h ; CHECK-NEXT: sunpklo z3.s, z0.h +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: sdivr z2.s, p0/m, z2.s, z3.s ; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h @@ -223,7 +223,6 @@ define <8 x i16> @srem_v8i16(<8 x i16> %op1, <8 x i16> %op2) { ; CHECK-NEXT: mov z3.d, z0.d ; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: sunpklo z4.s, z0.h -; CHECK-NEXT: ptrue p1.h, vl8 ; CHECK-NEXT: ext z2.b, z2.b, z1.b, #8 ; CHECK-NEXT: ext z3.b, z3.b, z0.b, #8 ; CHECK-NEXT: sunpklo z2.s, z2.h @@ -235,7 +234,8 @@ define <8 x i16> @srem_v8i16(<8 x i16> %op1, <8 x i16> %op2) { ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h ; CHECK-NEXT: splice z3.h, p0, z3.h, z2.h -; CHECK-NEXT: mls z0.h, p1/m, z3.h, z1.h +; CHECK-NEXT: ptrue p0.h, vl8 +; CHECK-NEXT: mls z0.h, p0/m, z3.h, z1.h ; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 ; CHECK-NEXT: ret %res = srem <8 x i16> %op1, %op2 @@ -248,7 +248,6 @@ define void @srem_v16i16(ptr %a, ptr %b) { ; CHECK-NEXT: ldp q4, q1, [x1] ; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldr q0, [x0, #16] -; CHECK-NEXT: ptrue p1.h, vl8 ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: mov z3.d, z0.d ; CHECK-NEXT: mov z5.d, z4.d @@ -277,9 +276,10 @@ define void @srem_v16i16(ptr %a, ptr %b) { ; CHECK-NEXT: splice z6.h, p0, z6.h, z5.h ; CHECK-NEXT: uzp1 z7.h, z7.h, z7.h ; CHECK-NEXT: splice z7.h, p0, z7.h, z2.h +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: movprfx z2, z3 -; CHECK-NEXT: mls z2.h, p1/m, z6.h, z4.h -; CHECK-NEXT: mls z0.h, p1/m, z7.h, z1.h +; CHECK-NEXT: mls z2.h, p0/m, z6.h, z4.h +; CHECK-NEXT: mls z0.h, p0/m, z7.h, z1.h ; CHECK-NEXT: stp q2, q0, [x0] ; CHECK-NEXT: ret %op1 = load <16 x i16>, ptr %a @@ -322,8 +322,8 @@ define <4 x i32> @srem_v4i32(<4 x i32> %op1, <4 x i32> %op2) { define void @srem_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: srem_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: movprfx z4, z1 ; CHECK-NEXT: sdiv z4.s, p0/m, z4.s, z0.s @@ -374,8 +374,8 @@ define <2 x i64> @srem_v2i64(<2 x i64> %op1, <2 x i64> %op2) { define void @srem_v4i64(ptr %a, ptr %b) { ; CHECK-LABEL: srem_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: movprfx z4, z1 ; CHECK-NEXT: sdiv z4.d, p0/m, z4.d, z0.d @@ -454,7 +454,6 @@ define <16 x i8> @urem_v16i8(<16 x i8> %op1, <16 x i8> %op2) { ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: mov z3.d, z0.d ; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ptrue p1.b, vl16 ; CHECK-NEXT: ext z2.b, z2.b, z1.b, #8 ; CHECK-NEXT: ext z3.b, z3.b, z0.b, #8 ; CHECK-NEXT: uunpklo z2.h, z2.b @@ -480,15 +479,16 @@ define <16 x i8> @urem_v16i8(<16 x i8> %op1, <16 x i8> %op2) { ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h ; CHECK-NEXT: udivr z3.s, p0/m, z3.s, z5.s ; CHECK-NEXT: ptrue p0.h, vl4 -; CHECK-NEXT: uzp1 z5.h, z6.h, z6.h ; CHECK-NEXT: splice z4.h, p0, z4.h, z2.h +; CHECK-NEXT: uzp1 z5.h, z6.h, z6.h ; CHECK-NEXT: uzp1 z2.b, z4.b, z4.b ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h ; CHECK-NEXT: splice z5.h, p0, z5.h, z3.h ; CHECK-NEXT: ptrue p0.b, vl8 ; CHECK-NEXT: uzp1 z3.b, z5.b, z5.b ; CHECK-NEXT: splice z3.b, p0, z3.b, z2.b -; CHECK-NEXT: mls z0.b, p1/m, z3.b, z1.b +; CHECK-NEXT: ptrue p0.b, vl16 +; CHECK-NEXT: mls z0.b, p0/m, z3.b, z1.b ; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 ; CHECK-NEXT: ret %res = urem <16 x i8> %op1, %op2 @@ -501,7 +501,6 @@ define void @urem_v32i8(ptr %a, ptr %b) { ; CHECK-NEXT: ldr q0, [x0, #16] ; CHECK-NEXT: ldr q1, [x1, #16] ; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ptrue p1.b, vl16 ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: mov z3.d, z0.d ; CHECK-NEXT: uunpklo z7.h, z1.b @@ -560,22 +559,23 @@ define void @urem_v32i8(ptr %a, ptr %b) { ; CHECK-NEXT: uzp1 z17.h, z17.h, z17.h ; CHECK-NEXT: udivr z18.s, p0/m, z18.s, z20.s ; CHECK-NEXT: ptrue p0.h, vl4 -; CHECK-NEXT: uzp1 z19.h, z21.h, z21.h ; CHECK-NEXT: splice z16.h, p0, z16.h, z17.h ; CHECK-NEXT: splice z2.h, p0, z2.h, z5.h ; CHECK-NEXT: splice z6.h, p0, z6.h, z7.h +; CHECK-NEXT: uzp1 z19.h, z21.h, z21.h ; CHECK-NEXT: uzp1 z5.b, z16.b, z16.b ; CHECK-NEXT: uzp1 z2.b, z2.b, z2.b ; CHECK-NEXT: uzp1 z6.b, z6.b, z6.b ; CHECK-NEXT: uzp1 z18.h, z18.h, z18.h ; CHECK-NEXT: splice z19.h, p0, z19.h, z18.h ; CHECK-NEXT: ptrue p0.b, vl8 -; CHECK-NEXT: uzp1 z7.b, z19.b, z19.b ; CHECK-NEXT: splice z6.b, p0, z6.b, z2.b +; CHECK-NEXT: uzp1 z7.b, z19.b, z19.b ; CHECK-NEXT: splice z7.b, p0, z7.b, z5.b -; CHECK-NEXT: mls z0.b, p1/m, z6.b, z1.b +; CHECK-NEXT: ptrue p0.b, vl16 +; CHECK-NEXT: mls z0.b, p0/m, z6.b, z1.b ; CHECK-NEXT: movprfx z2, z3 -; CHECK-NEXT: mls z2.b, p1/m, z7.b, z4.b +; CHECK-NEXT: mls z2.b, p0/m, z7.b, z4.b ; CHECK-NEXT: stp q2, q0, [x0] ; CHECK-NEXT: ret %op1 = load <32 x i8>, ptr %a @@ -588,11 +588,11 @@ define void @urem_v32i8(ptr %a, ptr %b) { define <4 x i16> @urem_v4i16(<4 x i16> %op1, <4 x i16> %op2) { ; CHECK-LABEL: urem_v4i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: uunpklo z2.s, z1.h ; CHECK-NEXT: uunpklo z3.s, z0.h +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: udivr z2.s, p0/m, z2.s, z3.s ; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h @@ -612,7 +612,6 @@ define <8 x i16> @urem_v8i16(<8 x i16> %op1, <8 x i16> %op2) { ; CHECK-NEXT: mov z3.d, z0.d ; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: uunpklo z4.s, z0.h -; CHECK-NEXT: ptrue p1.h, vl8 ; CHECK-NEXT: ext z2.b, z2.b, z1.b, #8 ; CHECK-NEXT: ext z3.b, z3.b, z0.b, #8 ; CHECK-NEXT: uunpklo z2.s, z2.h @@ -624,7 +623,8 @@ define <8 x i16> @urem_v8i16(<8 x i16> %op1, <8 x i16> %op2) { ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h ; CHECK-NEXT: splice z3.h, p0, z3.h, z2.h -; CHECK-NEXT: mls z0.h, p1/m, z3.h, z1.h +; CHECK-NEXT: ptrue p0.h, vl8 +; CHECK-NEXT: mls z0.h, p0/m, z3.h, z1.h ; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 ; CHECK-NEXT: ret %res = urem <8 x i16> %op1, %op2 @@ -637,7 +637,6 @@ define void @urem_v16i16(ptr %a, ptr %b) { ; CHECK-NEXT: ldp q4, q1, [x1] ; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldr q0, [x0, #16] -; CHECK-NEXT: ptrue p1.h, vl8 ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: mov z3.d, z0.d ; CHECK-NEXT: mov z5.d, z4.d @@ -666,9 +665,10 @@ define void @urem_v16i16(ptr %a, ptr %b) { ; CHECK-NEXT: splice z6.h, p0, z6.h, z5.h ; CHECK-NEXT: uzp1 z7.h, z7.h, z7.h ; CHECK-NEXT: splice z7.h, p0, z7.h, z2.h +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: movprfx z2, z3 -; CHECK-NEXT: mls z2.h, p1/m, z6.h, z4.h -; CHECK-NEXT: mls z0.h, p1/m, z7.h, z1.h +; CHECK-NEXT: mls z2.h, p0/m, z6.h, z4.h +; CHECK-NEXT: mls z0.h, p0/m, z7.h, z1.h ; CHECK-NEXT: stp q2, q0, [x0] ; CHECK-NEXT: ret %op1 = load <16 x i16>, ptr %a @@ -711,8 +711,8 @@ define <4 x i32> @urem_v4i32(<4 x i32> %op1, <4 x i32> %op2) { define void @urem_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: urem_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: movprfx z4, z1 ; CHECK-NEXT: udiv z4.s, p0/m, z4.s, z0.s @@ -763,8 +763,8 @@ define <2 x i64> @urem_v2i64(<2 x i64> %op1, <2 x i64> %op2) { define void @urem_v4i64(ptr %a, ptr %b) { ; CHECK-LABEL: urem_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: movprfx z4, z1 ; CHECK-NEXT: udiv z4.d, p0/m, z4.d, z0.d diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-select.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-select.ll index 3b58e35bd844..906112f7ac39 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-select.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-select.ll @@ -7,8 +7,8 @@ target triple = "aarch64-unknown-linux-gnu" define <4 x i8> @select_v4i8(<4 x i8> %op1, <4 x i8> %op2, i1 %mask) { ; CHECK-LABEL: select_v4i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w0 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 ; CHECK-NEXT: and z2.h, z2.h, #0x1 @@ -23,8 +23,8 @@ define <4 x i8> @select_v4i8(<4 x i8> %op1, <4 x i8> %op2, i1 %mask) { define <8 x i8> @select_v8i8(<8 x i8> %op1, <8 x i8> %op2, i1 %mask) { ; CHECK-LABEL: select_v8i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z2.b, w0 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 ; CHECK-NEXT: cmpne p0.b, p0/z, z2.b, #0 @@ -38,8 +38,8 @@ define <8 x i8> @select_v8i8(<8 x i8> %op1, <8 x i8> %op2, i1 %mask) { define <16 x i8> @select_v16i8(<16 x i8> %op1, <16 x i8> %op2, i1 %mask) { ; CHECK-LABEL: select_v16i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z2.b, w0 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 ; CHECK-NEXT: cmpne p0.b, p0/z, z2.b, #0 @@ -53,8 +53,8 @@ define <16 x i8> @select_v16i8(<16 x i8> %op1, <16 x i8> %op2, i1 %mask) { define void @select_v32i8(ptr %a, ptr %b, i1 %mask) { ; CHECK-LABEL: select_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z0.b, w2 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: cmpne p0.b, p0/z, z0.b, #0 ; CHECK-NEXT: ldr q0, [x0] ; CHECK-NEXT: ldr q1, [x0, #16] @@ -74,8 +74,8 @@ define void @select_v32i8(ptr %a, ptr %b, i1 %mask) { define <2 x i16> @select_v2i16(<2 x i16> %op1, <2 x i16> %op2, i1 %mask) { ; CHECK-LABEL: select_v2i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and w8, w0, #0x1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 ; CHECK-NEXT: mov z2.s, w8 @@ -90,8 +90,8 @@ define <2 x i16> @select_v2i16(<2 x i16> %op1, <2 x i16> %op2, i1 %mask) { define <4 x i16> @select_v4i16(<4 x i16> %op1, <4 x i16> %op2, i1 %mask) { ; CHECK-LABEL: select_v4i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w0 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 ; CHECK-NEXT: and z2.h, z2.h, #0x1 @@ -106,8 +106,8 @@ define <4 x i16> @select_v4i16(<4 x i16> %op1, <4 x i16> %op2, i1 %mask) { define <8 x i16> @select_v8i16(<8 x i16> %op1, <8 x i16> %op2, i1 %mask) { ; CHECK-LABEL: select_v8i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w0 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 ; CHECK-NEXT: and z2.h, z2.h, #0x1 @@ -122,8 +122,8 @@ define <8 x i16> @select_v8i16(<8 x i16> %op1, <8 x i16> %op2, i1 %mask) { define void @select_v16i16(ptr %a, ptr %b, i1 %mask) { ; CHECK-LABEL: select_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z0.h, w2 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: and z0.h, z0.h, #0x1 ; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #0 ; CHECK-NEXT: ldr q0, [x0] @@ -144,8 +144,8 @@ define void @select_v16i16(ptr %a, ptr %b, i1 %mask) { define <2 x i32> @select_v2i32(<2 x i32> %op1, <2 x i32> %op2, i1 %mask) { ; CHECK-LABEL: select_v2i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and w8, w0, #0x1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 ; CHECK-NEXT: mov z2.s, w8 @@ -160,8 +160,8 @@ define <2 x i32> @select_v2i32(<2 x i32> %op1, <2 x i32> %op2, i1 %mask) { define <4 x i32> @select_v4i32(<4 x i32> %op1, <4 x i32> %op2, i1 %mask) { ; CHECK-LABEL: select_v4i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and w8, w0, #0x1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 ; CHECK-NEXT: mov z2.s, w8 @@ -176,8 +176,8 @@ define <4 x i32> @select_v4i32(<4 x i32> %op1, <4 x i32> %op2, i1 %mask) { define void @select_v8i32(ptr %a, ptr %b, i1 %mask) { ; CHECK-LABEL: select_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and w8, w2, #0x1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z0.s, w8 ; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0 ; CHECK-NEXT: ldr q0, [x0] @@ -198,9 +198,9 @@ define void @select_v8i32(ptr %a, ptr %b, i1 %mask) { define <1 x i64> @select_v1i64(<1 x i64> %op1, <1 x i64> %op2, i1 %mask) { ; CHECK-LABEL: select_v1i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 ; CHECK-NEXT: and x8, x0, #0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 ; CHECK-NEXT: mov z2.d, x8 @@ -215,9 +215,9 @@ define <1 x i64> @select_v1i64(<1 x i64> %op1, <1 x i64> %op2, i1 %mask) { define <2 x i64> @select_v2i64(<2 x i64> %op1, <2 x i64> %op2, i1 %mask) { ; CHECK-LABEL: select_v2i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 ; CHECK-NEXT: and x8, x0, #0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 ; CHECK-NEXT: mov z2.d, x8 @@ -232,9 +232,9 @@ define <2 x i64> @select_v2i64(<2 x i64> %op1, <2 x i64> %op2, i1 %mask) { define void @select_v4i64(ptr %a, ptr %b, i1 %mask) { ; CHECK-LABEL: select_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $w2 killed $w2 def $x2 ; CHECK-NEXT: and x8, x2, #0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z0.d, x8 ; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 ; CHECK-NEXT: ldr q0, [x0] diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-shifts.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-shifts.ll index c7fa0e8ad5e4..9ed52e321d9a 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-shifts.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-shifts.ll @@ -52,8 +52,8 @@ define <16 x i8> @ashr_v16i8(<16 x i8> %op1, <16 x i8> %op2) { define void @ashr_v32i8(ptr %a, ptr %b) { ; CHECK-LABEL: ashr_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: asrr z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: movprfx z1, z2 @@ -111,8 +111,8 @@ define <8 x i16> @ashr_v8i16(<8 x i16> %op1, <8 x i16> %op2) { define void @ashr_v16i16(ptr %a, ptr %b) { ; CHECK-LABEL: ashr_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: asrr z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -155,8 +155,8 @@ define <4 x i32> @ashr_v4i32(<4 x i32> %op1, <4 x i32> %op2) { define void @ashr_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: ashr_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: asrr z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -199,8 +199,8 @@ define <2 x i64> @ashr_v2i64(<2 x i64> %op1, <2 x i64> %op2) { define void @ashr_v4i64(ptr %a, ptr %b) { ; CHECK-LABEL: ashr_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: asrr z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 @@ -221,9 +221,9 @@ define void @ashr_v4i64(ptr %a, ptr %b) { define <4 x i8> @lshr_v4i8(<4 x i8> %op1, <4 x i8> %op2) { ; CHECK-LABEL: lshr_v4i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: and z1.h, z1.h, #0xff ; CHECK-NEXT: and z0.h, z0.h, #0xff ; CHECK-NEXT: lsr z0.h, p0/m, z0.h, z1.h @@ -262,8 +262,8 @@ define <16 x i8> @lshr_v16i8(<16 x i8> %op1, <16 x i8> %op2) { define void @lshr_v32i8(ptr %a, ptr %b) { ; CHECK-LABEL: lshr_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: lsrr z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: movprfx z1, z2 @@ -280,9 +280,9 @@ define void @lshr_v32i8(ptr %a, ptr %b) { define <2 x i16> @lshr_v2i16(<2 x i16> %op1, <2 x i16> %op2) { ; CHECK-LABEL: lshr_v2i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: and z1.s, z1.s, #0xffff ; CHECK-NEXT: and z0.s, z0.s, #0xffff ; CHECK-NEXT: lsr z0.s, p0/m, z0.s, z1.s @@ -321,8 +321,8 @@ define <8 x i16> @lshr_v8i16(<8 x i16> %op1, <8 x i16> %op2) { define void @lshr_v16i16(ptr %a, ptr %b) { ; CHECK-LABEL: lshr_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: lsrr z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -365,8 +365,8 @@ define <4 x i32> @lshr_v4i32(<4 x i32> %op1, <4 x i32> %op2) { define void @lshr_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: lshr_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: lsrr z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -409,8 +409,8 @@ define <2 x i64> @lshr_v2i64(<2 x i64> %op1, <2 x i64> %op2) { define void @lshr_v4i64(ptr %a, ptr %b) { ; CHECK-LABEL: lshr_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: lsrr z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 @@ -431,8 +431,8 @@ define void @lshr_v4i64(ptr %a, ptr %b) { define <2 x i8> @shl_v2i8(<2 x i8> %op1, <2 x i8> %op2) { ; CHECK-LABEL: shl_v2i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 +; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: and z1.s, z1.s, #0xff ; CHECK-NEXT: lsl z0.s, p0/m, z0.s, z1.s @@ -445,8 +445,8 @@ define <2 x i8> @shl_v2i8(<2 x i8> %op1, <2 x i8> %op2) { define <4 x i8> @shl_v4i8(<4 x i8> %op1, <4 x i8> %op2) { ; CHECK-LABEL: shl_v4i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 +; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: and z1.h, z1.h, #0xff ; CHECK-NEXT: lsl z0.h, p0/m, z0.h, z1.h @@ -485,8 +485,8 @@ define <16 x i8> @shl_v16i8(<16 x i8> %op1, <16 x i8> %op2) { define void @shl_v32i8(ptr %a, ptr %b) { ; CHECK-LABEL: shl_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: lslr z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: movprfx z1, z2 @@ -529,8 +529,8 @@ define <8 x i16> @shl_v8i16(<8 x i16> %op1, <8 x i16> %op2) { define void @shl_v16i16(ptr %a, ptr %b) { ; CHECK-LABEL: shl_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: lslr z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -573,8 +573,8 @@ define <4 x i32> @shl_v4i32(<4 x i32> %op1, <4 x i32> %op2) { define void @shl_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: shl_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: lslr z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -617,8 +617,8 @@ define <2 x i64> @shl_v2i64(<2 x i64> %op1, <2 x i64> %op2) { define void @shl_v4i64(ptr %a, ptr %b) { ; CHECK-LABEL: shl_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: lslr z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-to-fp.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-to-fp.ll index 5c5cf68135bf..b285659258f3 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-to-fp.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-to-fp.ll @@ -36,8 +36,8 @@ define void @ucvtf_v8i16_v8f16(ptr %a, ptr %b) { define void @ucvtf_v16i16_v16f16(ptr %a, ptr %b) { ; CHECK-LABEL: ucvtf_v16i16_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ucvtf z0.h, p0/m, z0.h ; CHECK-NEXT: ucvtf z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x1] @@ -55,8 +55,8 @@ define void @ucvtf_v16i16_v16f16(ptr %a, ptr %b) { define <2 x float> @ucvtf_v2i16_v2f32(<2 x i16> %op1) { ; CHECK-LABEL: ucvtf_v2i16_v2f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: and z0.s, z0.s, #0xffff ; CHECK-NEXT: ucvtf z0.s, p0/m, z0.s ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 @@ -68,8 +68,8 @@ define <2 x float> @ucvtf_v2i16_v2f32(<2 x i16> %op1) { define <4 x float> @ucvtf_v4i16_v4f32(<4 x i16> %op1) { ; CHECK-LABEL: ucvtf_v4i16_v4f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: ucvtf z0.s, p0/m, z0.s ; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 @@ -277,8 +277,8 @@ define <4 x half> @ucvtf_v4i32_v4f16(<4 x i32> %op1) { define <8 x half> @ucvtf_v8i32_v8f16(ptr %a) { ; CHECK-LABEL: ucvtf_v8i32_v8f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ucvtf z1.h, p0/m, z1.s ; CHECK-NEXT: ucvtf z0.h, p0/m, z0.s ; CHECK-NEXT: ptrue p0.h, vl4 @@ -295,8 +295,8 @@ define <8 x half> @ucvtf_v8i32_v8f16(ptr %a) { define void @ucvtf_v16i32_v16f16(ptr %a, ptr %b) { ; CHECK-LABEL: ucvtf_v16i32_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0, #32] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: ucvtf z1.h, p0/m, z1.s ; CHECK-NEXT: ucvtf z0.h, p0/m, z0.s @@ -348,8 +348,8 @@ define <4 x float> @ucvtf_v4i32_v4f32(<4 x i32> %op1) { define void @ucvtf_v8i32_v8f32(ptr %a, ptr %b) { ; CHECK-LABEL: ucvtf_v8i32_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ucvtf z0.s, p0/m, z0.s ; CHECK-NEXT: ucvtf z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x1] @@ -367,8 +367,8 @@ define void @ucvtf_v8i32_v8f32(ptr %a, ptr %b) { define <2 x double> @ucvtf_v2i32_v2f64(<2 x i32> %op1) { ; CHECK-LABEL: ucvtf_v2i32_v2f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: ucvtf z0.d, p0/m, z0.d ; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 @@ -446,16 +446,16 @@ define <2 x half> @ucvtf_v2i64_v2f16(<2 x i64> %op1) { define <4 x half> @ucvtf_v4i64_v4f16(ptr %a) { ; CHECK-LABEL: ucvtf_v4i64_v4f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] -; CHECK-NEXT: ptrue p1.s +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ucvtf z1.s, p0/m, z1.d ; CHECK-NEXT: ucvtf z0.s, p0/m, z0.d ; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: uzp1 z1.s, z1.s, z1.s ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s ; CHECK-NEXT: splice z0.s, p0, z0.s, z1.s -; CHECK-NEXT: fcvt z0.h, p1/m, z0.s +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: fcvt z0.h, p0/m, z0.s ; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 ; CHECK-NEXT: ret @@ -467,10 +467,9 @@ define <4 x half> @ucvtf_v4i64_v4f16(ptr %a) { define <8 x half> @ucvtf_v8i64_v8f16(ptr %a) { ; CHECK-LABEL: ucvtf_v8i64_v8f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0, #32] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q2, q3, [x0] -; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: ucvtf z0.s, p0/m, z0.d ; CHECK-NEXT: ucvtf z1.s, p0/m, z1.d ; CHECK-NEXT: ucvtf z3.s, p0/m, z3.d @@ -482,11 +481,12 @@ define <8 x half> @ucvtf_v8i64_v8f16(ptr %a) { ; CHECK-NEXT: uzp1 z2.s, z2.s, z2.s ; CHECK-NEXT: splice z1.s, p0, z1.s, z0.s ; CHECK-NEXT: splice z2.s, p0, z2.s, z3.s -; CHECK-NEXT: ptrue p0.h, vl4 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: movprfx z0, z1 -; CHECK-NEXT: fcvt z0.h, p1/m, z1.s +; CHECK-NEXT: fcvt z0.h, p0/m, z1.s ; CHECK-NEXT: movprfx z1, z2 -; CHECK-NEXT: fcvt z1.h, p1/m, z2.s +; CHECK-NEXT: fcvt z1.h, p0/m, z2.s +; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: uzp1 z2.h, z0.h, z0.h ; CHECK-NEXT: uzp1 z0.h, z1.h, z1.h ; CHECK-NEXT: splice z0.h, p0, z0.h, z2.h @@ -517,8 +517,8 @@ define <2 x float> @ucvtf_v2i64_v2f32(<2 x i64> %op1) { define <4 x float> @ucvtf_v4i64_v4f32(ptr %a) { ; CHECK-LABEL: ucvtf_v4i64_v4f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ucvtf z1.s, p0/m, z1.d ; CHECK-NEXT: ucvtf z0.s, p0/m, z0.d ; CHECK-NEXT: ptrue p0.s, vl2 @@ -535,8 +535,8 @@ define <4 x float> @ucvtf_v4i64_v4f32(ptr %a) { define void @ucvtf_v8i64_v8f32(ptr %a, ptr %b) { ; CHECK-LABEL: ucvtf_v8i64_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0, #32] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: ucvtf z1.s, p0/m, z1.d ; CHECK-NEXT: ucvtf z0.s, p0/m, z0.d @@ -576,8 +576,8 @@ define <2 x double> @ucvtf_v2i64_v2f64(<2 x i64> %op1) { define void @ucvtf_v4i64_v4f64(ptr %a, ptr %b) { ; CHECK-LABEL: ucvtf_v4i64_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ucvtf z0.d, p0/m, z0.d ; CHECK-NEXT: ucvtf z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x1] @@ -621,8 +621,8 @@ define void @scvtf_v8i16_v8f16(ptr %a, ptr %b) { define void @scvtf_v16i16_v16f16(ptr %a, ptr %b) { ; CHECK-LABEL: scvtf_v16i16_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: scvtf z0.h, p0/m, z0.h ; CHECK-NEXT: scvtf z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x1] @@ -652,8 +652,8 @@ define <2 x float> @scvtf_v2i16_v2f32(<2 x i16> %op1) { define <4 x float> @scvtf_v4i16_v4f32(<4 x i16> %op1) { ; CHECK-LABEL: scvtf_v4i16_v4f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: sunpklo z0.s, z0.h ; CHECK-NEXT: scvtf z0.s, p0/m, z0.s ; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 @@ -850,8 +850,8 @@ define <4 x half> @scvtf_v4i32_v4f16(<4 x i32> %op1) { define <8 x half> @scvtf_v8i32_v8f16(ptr %a) { ; CHECK-LABEL: scvtf_v8i32_v8f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: scvtf z1.h, p0/m, z1.s ; CHECK-NEXT: scvtf z0.h, p0/m, z0.s ; CHECK-NEXT: ptrue p0.h, vl4 @@ -896,8 +896,8 @@ define <4 x float> @scvtf_v4i32_v4f32(<4 x i32> %op1) { define void @scvtf_v8i32_v8f32(ptr %a, ptr %b) { ; CHECK-LABEL: scvtf_v8i32_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: scvtf z0.s, p0/m, z0.s ; CHECK-NEXT: scvtf z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x1] @@ -915,8 +915,8 @@ define void @scvtf_v8i32_v8f32(ptr %a, ptr %b) { define <2 x double> @scvtf_v2i32_v2f64(<2 x i32> %op1) { ; CHECK-LABEL: scvtf_v2i32_v2f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: sunpklo z0.d, z0.s ; CHECK-NEXT: scvtf z0.d, p0/m, z0.d ; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 @@ -1038,16 +1038,16 @@ define <2 x half> @scvtf_v2i64_v2f16(<2 x i64> %op1) { define <4 x half> @scvtf_v4i64_v4f16(ptr %a) { ; CHECK-LABEL: scvtf_v4i64_v4f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] -; CHECK-NEXT: ptrue p1.s +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: scvtf z1.s, p0/m, z1.d ; CHECK-NEXT: scvtf z0.s, p0/m, z0.d ; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: uzp1 z1.s, z1.s, z1.s ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s ; CHECK-NEXT: splice z0.s, p0, z0.s, z1.s -; CHECK-NEXT: fcvt z0.h, p1/m, z0.s +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: fcvt z0.h, p0/m, z0.s ; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 ; CHECK-NEXT: ret @@ -1076,8 +1076,8 @@ define <2 x float> @scvtf_v2i64_v2f32(<2 x i64> %op1) { define <4 x float> @scvtf_v4i64_v4f32(ptr %a) { ; CHECK-LABEL: scvtf_v4i64_v4f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: scvtf z1.s, p0/m, z1.d ; CHECK-NEXT: scvtf z0.s, p0/m, z0.d ; CHECK-NEXT: ptrue p0.s, vl2 @@ -1110,8 +1110,8 @@ define <2 x double> @scvtf_v2i64_v2f64(<2 x i64> %op1) { define void @scvtf_v4i64_v4f64(ptr %a, ptr %b) { ; CHECK-LABEL: scvtf_v4i64_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: scvtf z0.d, p0/m, z0.d ; CHECK-NEXT: scvtf z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x1] diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-vselect.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-vselect.ll index 1809cfcf3db6..81bbaa92d4b4 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-vselect.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-vselect.ll @@ -61,8 +61,8 @@ define <16 x i8> @select_v16i8(<16 x i8> %op1, <16 x i8> %op2, <16 x i1> %mask) define void @select_v32i8(ptr %a, ptr %b) { ; CHECK-LABEL: select_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q0, q2, [x0] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q3, [x1] ; CHECK-NEXT: cmpeq p1.b, p0/z, z0.b, z1.b ; CHECK-NEXT: cmpeq p0.b, p0/z, z2.b, z3.b @@ -136,8 +136,8 @@ define <8 x i16> @select_v8i16(<8 x i16> %op1, <8 x i16> %op2, <8 x i1> %mask) { define void @select_v16i16(ptr %a, ptr %b) { ; CHECK-LABEL: select_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q2, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q3, [x1] ; CHECK-NEXT: cmpeq p1.h, p0/z, z0.h, z1.h ; CHECK-NEXT: cmpeq p0.h, p0/z, z2.h, z3.h @@ -193,8 +193,8 @@ define <4 x i32> @select_v4i32(<4 x i32> %op1, <4 x i32> %op2, <4 x i1> %mask) { define void @select_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: select_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q2, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q3, [x1] ; CHECK-NEXT: cmpeq p1.s, p0/z, z0.s, z1.s ; CHECK-NEXT: cmpeq p0.s, p0/z, z2.s, z3.s @@ -213,9 +213,9 @@ define void @select_v8i32(ptr %a, ptr %b) { define <1 x i64> @select_v1i64(<1 x i64> %op1, <1 x i64> %op2, <1 x i1> %mask) { ; CHECK-LABEL: select_v1i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 ; CHECK-NEXT: and x8, x0, #0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 ; CHECK-NEXT: mov z2.d, x8 @@ -249,8 +249,8 @@ define <2 x i64> @select_v2i64(<2 x i64> %op1, <2 x i64> %op2, <2 x i1> %mask) { define void @select_v4i64(ptr %a, ptr %b) { ; CHECK-LABEL: select_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q2, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q3, [x1] ; CHECK-NEXT: cmpeq p1.d, p0/z, z0.d, z1.d ; CHECK-NEXT: cmpeq p0.d, p0/z, z2.d, z3.d diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-log-reduce.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-log-reduce.ll index bb1bd8fe72b2..c4aeb4465c53 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-log-reduce.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-log-reduce.ll @@ -48,8 +48,8 @@ define i8 @andv_v16i8(<16 x i8> %a) { define i8 @andv_v32i8(ptr %a) { ; CHECK-LABEL: andv_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: and z0.d, z1.d, z0.d ; CHECK-NEXT: andv b0, p0, z0.b ; CHECK-NEXT: fmov w0, s0 @@ -98,8 +98,8 @@ define i16 @andv_v8i16(<8 x i16> %a) { define i16 @andv_v16i16(ptr %a) { ; CHECK-LABEL: andv_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: and z0.d, z1.d, z0.d ; CHECK-NEXT: andv h0, p0, z0.h ; CHECK-NEXT: fmov w0, s0 @@ -136,8 +136,8 @@ define i32 @andv_v4i32(<4 x i32> %a) { define i32 @andv_v8i32(ptr %a) { ; CHECK-LABEL: andv_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: and z0.d, z1.d, z0.d ; CHECK-NEXT: andv s0, p0, z0.s ; CHECK-NEXT: fmov w0, s0 @@ -162,8 +162,8 @@ define i64 @andv_v2i64(<2 x i64> %a) { define i64 @andv_v4i64(ptr %a) { ; CHECK-LABEL: andv_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: and z0.d, z1.d, z0.d ; CHECK-NEXT: andv d0, p0, z0.d ; CHECK-NEXT: fmov x0, d0 @@ -216,8 +216,8 @@ define i8 @eorv_v16i8(<16 x i8> %a) { define i8 @eorv_v32i8(ptr %a) { ; CHECK-LABEL: eorv_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: eor z0.d, z1.d, z0.d ; CHECK-NEXT: eorv b0, p0, z0.b ; CHECK-NEXT: fmov w0, s0 @@ -266,8 +266,8 @@ define i16 @eorv_v8i16(<8 x i16> %a) { define i16 @eorv_v16i16(ptr %a) { ; CHECK-LABEL: eorv_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: eor z0.d, z1.d, z0.d ; CHECK-NEXT: eorv h0, p0, z0.h ; CHECK-NEXT: fmov w0, s0 @@ -304,8 +304,8 @@ define i32 @eorv_v4i32(<4 x i32> %a) { define i32 @eorv_v8i32(ptr %a) { ; CHECK-LABEL: eorv_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: eor z0.d, z1.d, z0.d ; CHECK-NEXT: eorv s0, p0, z0.s ; CHECK-NEXT: fmov w0, s0 @@ -330,8 +330,8 @@ define i64 @eorv_v2i64(<2 x i64> %a) { define i64 @eorv_v4i64(ptr %a) { ; CHECK-LABEL: eorv_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: eor z0.d, z1.d, z0.d ; CHECK-NEXT: eorv d0, p0, z0.d ; CHECK-NEXT: fmov x0, d0 @@ -384,8 +384,8 @@ define i8 @orv_v16i8(<16 x i8> %a) { define i8 @orv_v32i8(ptr %a) { ; CHECK-LABEL: orv_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: orr z0.d, z1.d, z0.d ; CHECK-NEXT: orv b0, p0, z0.b ; CHECK-NEXT: fmov w0, s0 @@ -434,8 +434,8 @@ define i16 @orv_v8i16(<8 x i16> %a) { define i16 @orv_v16i16(ptr %a) { ; CHECK-LABEL: orv_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: orr z0.d, z1.d, z0.d ; CHECK-NEXT: orv h0, p0, z0.h ; CHECK-NEXT: fmov w0, s0 @@ -472,8 +472,8 @@ define i32 @orv_v4i32(<4 x i32> %a) { define i32 @orv_v8i32(ptr %a) { ; CHECK-LABEL: orv_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: orr z0.d, z1.d, z0.d ; CHECK-NEXT: orv s0, p0, z0.s ; CHECK-NEXT: fmov w0, s0 @@ -498,8 +498,8 @@ define i64 @orv_v2i64(<2 x i64> %a) { define i64 @orv_v4i64(ptr %a) { ; CHECK-LABEL: orv_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: orr z0.d, z1.d, z0.d ; CHECK-NEXT: orv d0, p0, z0.d ; CHECK-NEXT: fmov x0, d0 diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-store.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-store.ll index e81270674474..f2b3f9b12ea7 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-store.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-store.ll @@ -123,8 +123,8 @@ define void @masked_store_v32i8(ptr %dst, <32 x i1> %mask) { ; CHECK-NEXT: asr z0.b, z0.b, #7 ; CHECK-NEXT: asr z1.b, z1.b, #7 ; CHECK-NEXT: cmpne p1.b, p0/z, z0.b, #0 -; CHECK-NEXT: mov z0.b, #0 // =0x0 ; CHECK-NEXT: cmpne p0.b, p0/z, z1.b, #0 +; CHECK-NEXT: mov z0.b, #0 // =0x0 ; CHECK-NEXT: st1b { z0.b }, p1, [x0, x8] ; CHECK-NEXT: st1b { z0.b }, p0, [x0] ; CHECK-NEXT: add sp, sp, #32 @@ -204,8 +204,8 @@ define void @masked_store_v16f16(ptr %dst, <16 x i1> %mask) { ; CHECK-NEXT: asr z0.h, z0.h, #15 ; CHECK-NEXT: asr z1.h, z1.h, #15 ; CHECK-NEXT: cmpne p1.h, p0/z, z1.h, #0 -; CHECK-NEXT: mov z1.h, #0 // =0x0 ; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #0 +; CHECK-NEXT: mov z1.h, #0 // =0x0 ; CHECK-NEXT: st1h { z1.h }, p1, [x0, x8, lsl #1] ; CHECK-NEXT: st1h { z1.h }, p0, [x0] ; CHECK-NEXT: ret @@ -242,7 +242,7 @@ define void @masked_store_v8f32(ptr %dst, <8 x i1> %mask) { ; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: fmov w8, s1 ; CHECK-NEXT: fmov w9, s2 -; CHECK-NEXT: mov z2.s, #0 // =0x0 +; CHECK-NEXT: mov z2.b, z0.b[3] ; CHECK-NEXT: strh w8, [sp, #14] ; CHECK-NEXT: fmov w8, s3 ; CHECK-NEXT: mov z3.b, z0.b[2] @@ -258,9 +258,9 @@ define void @masked_store_v8f32(ptr %dst, <8 x i1> %mask) { ; CHECK-NEXT: lsl z1.s, z1.s, #31 ; CHECK-NEXT: asr z1.s, z1.s, #31 ; CHECK-NEXT: cmpne p1.s, p0/z, z1.s, #0 -; CHECK-NEXT: mov z1.b, z0.b[3] -; CHECK-NEXT: st1w { z2.s }, p1, [x0, x8, lsl #2] -; CHECK-NEXT: fmov w8, s1 +; CHECK-NEXT: mov z1.s, #0 // =0x0 +; CHECK-NEXT: st1w { z1.s }, p1, [x0, x8, lsl #2] +; CHECK-NEXT: fmov w8, s2 ; CHECK-NEXT: strh w9, [sp] ; CHECK-NEXT: strh w8, [sp, #6] ; CHECK-NEXT: fmov w8, s3 @@ -272,7 +272,7 @@ define void @masked_store_v8f32(ptr %dst, <8 x i1> %mask) { ; CHECK-NEXT: lsl z0.s, z0.s, #31 ; CHECK-NEXT: asr z0.s, z0.s, #31 ; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0 -; CHECK-NEXT: st1w { z2.s }, p0, [x0] +; CHECK-NEXT: st1w { z1.s }, p0, [x0] ; CHECK-NEXT: add sp, sp, #16 ; CHECK-NEXT: ret call void @llvm.masked.store.v8f32(<8 x float> zeroinitializer, ptr %dst, i32 8, <8 x i1> %mask) @@ -310,8 +310,8 @@ define void @masked_store_v4f64(ptr %dst, <4 x i1> %mask) { ; CHECK-NEXT: asr z1.d, z1.d, #63 ; CHECK-NEXT: asr z0.d, z0.d, #63 ; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0 -; CHECK-NEXT: mov z0.d, #0 // =0x0 ; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0 +; CHECK-NEXT: mov z0.d, #0 // =0x0 ; CHECK-NEXT: st1d { z0.d }, p1, [x0, x8, lsl #3] ; CHECK-NEXT: st1d { z0.d }, p0, [x0] ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-optimize-ptrue.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-optimize-ptrue.ll index f0b0b3269e98..6fcb95f28333 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-optimize-ptrue.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-optimize-ptrue.ll @@ -170,8 +170,8 @@ define void @abs_v4i32(ptr %a) { define void @abs_v8i32(ptr %a) { ; CHECK-LABEL: abs_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: abs z0.s, p0/m, z0.s ; CHECK-NEXT: abs z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -199,8 +199,8 @@ define void @abs_v2i64(ptr %a) { define void @abs_v4i64(ptr %a) { ; CHECK-LABEL: abs_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: abs z0.d, p0/m, z0.d ; CHECK-NEXT: abs z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] @@ -263,8 +263,8 @@ define void @fadd_v8f16(ptr %a, ptr %b) { define void @fadd_v16f16(ptr %a, ptr %b) { ; CHECK-LABEL: fadd_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fadd z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -313,8 +313,8 @@ define void @fadd_v4f32(ptr %a, ptr %b) { define void @fadd_v8f32(ptr %a, ptr %b) { ; CHECK-LABEL: fadd_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fadd z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -347,8 +347,8 @@ define void @fadd_v2f64(ptr %a, ptr %b) { define void @fadd_v4f64(ptr %a, ptr %b) { ; CHECK-LABEL: fadd_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fadd z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-rev.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-rev.ll index d1bff4fa21a1..00413302798c 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-rev.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-rev.ll @@ -9,8 +9,8 @@ target triple = "aarch64-unknown-linux-gnu" define void @test_revbv16i16(ptr %a) { ; CHECK-LABEL: test_revbv16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: revb z0.h, p0/m, z0.h ; CHECK-NEXT: revb z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x0] @@ -25,8 +25,8 @@ define void @test_revbv16i16(ptr %a) { define void @test_revbv8i32(ptr %a) { ; CHECK-LABEL: test_revbv8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: revb z0.s, p0/m, z0.s ; CHECK-NEXT: revb z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -41,8 +41,8 @@ define void @test_revbv8i32(ptr %a) { define void @test_revbv4i64(ptr %a) { ; CHECK-LABEL: test_revbv4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: revb z0.d, p0/m, z0.d ; CHECK-NEXT: revb z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] @@ -57,8 +57,8 @@ define void @test_revbv4i64(ptr %a) { define void @test_revhv8i32(ptr %a) { ; CHECK-LABEL: test_revhv8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: revh z0.s, p0/m, z0.s ; CHECK-NEXT: revh z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -73,8 +73,8 @@ define void @test_revhv8i32(ptr %a) { define void @test_revhv8f32(ptr %a) { ; CHECK-LABEL: test_revhv8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: revh z0.s, p0/m, z0.s ; CHECK-NEXT: revh z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -89,8 +89,8 @@ define void @test_revhv8f32(ptr %a) { define void @test_revhv4i64(ptr %a) { ; CHECK-LABEL: test_revhv4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: revh z0.d, p0/m, z0.d ; CHECK-NEXT: revh z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] @@ -105,8 +105,8 @@ define void @test_revhv4i64(ptr %a) { define void @test_revwv4i64(ptr %a) { ; CHECK-LABEL: test_revwv4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: revw z0.d, p0/m, z0.d ; CHECK-NEXT: revw z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] @@ -121,8 +121,8 @@ define void @test_revwv4i64(ptr %a) { define void @test_revwv4f64(ptr %a) { ; CHECK-LABEL: test_revwv4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: revw z0.d, p0/m, z0.d ; CHECK-NEXT: revw z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] @@ -150,8 +150,8 @@ define <16 x i8> @test_revv16i8(ptr %a) { define void @test_revwv8i32v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: test_revwv8i32v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldp q0, q1, [x1] +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: revw z0.d, p0/m, z0.d ; CHECK-NEXT: revw z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] @@ -166,8 +166,8 @@ define void @test_revwv8i32v8i32(ptr %a, ptr %b) { define void @test_revhv32i16(ptr %a) { ; CHECK-LABEL: test_revhv32i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldp q0, q1, [x0, #32] +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: revh z0.d, p0/m, z0.d ; CHECK-NEXT: revh z1.d, p0/m, z1.d @@ -202,8 +202,8 @@ define void @test_rev_elts_fail(ptr %a) { define void @test_revdv4i64_sve2p1(ptr %a) #1 { ; CHECK-LABEL: test_revdv4i64_sve2p1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: revd z0.q, p0/m, z0.q ; CHECK-NEXT: revd z1.q, p0/m, z1.q ; CHECK-NEXT: stp q0, q1, [x0] @@ -217,8 +217,8 @@ define void @test_revdv4i64_sve2p1(ptr %a) #1 { define void @test_revdv4f64_sve2p1(ptr %a) #1 { ; CHECK-LABEL: test_revdv4f64_sve2p1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: revd z0.q, p0/m, z0.q ; CHECK-NEXT: revd z1.q, p0/m, z1.q ; CHECK-NEXT: stp q0, q1, [x0] diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-zip-uzp-trn.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-zip-uzp-trn.ll index d7bfb6b2680e..cb73030306b0 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-zip-uzp-trn.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-zip-uzp-trn.ll @@ -82,11 +82,11 @@ define void @zip_v32i16(ptr %a, ptr %b) { ; CHECK-NEXT: .cfi_def_cfa_offset 64 ; CHECK-NEXT: ldp q1, q3, [x1] ; CHECK-NEXT: ldp q0, q4, [x0] -; CHECK-NEXT: ldp q2, q6, [x0, #32] +; CHECK-NEXT: ldp q2, q5, [x0, #32] ; CHECK-NEXT: mov z16.h, z3.h[7] ; CHECK-NEXT: mov z18.h, z3.h[6] ; CHECK-NEXT: mov z17.h, z4.h[7] -; CHECK-NEXT: ldp q5, q7, [x1, #32] +; CHECK-NEXT: ldp q6, q7, [x1, #32] ; CHECK-NEXT: mov z19.h, z4.h[6] ; CHECK-NEXT: fmov w8, s16 ; CHECK-NEXT: mov z16.h, z3.h[5] @@ -98,13 +98,13 @@ define void @zip_v32i16(ptr %a, ptr %b) { ; CHECK-NEXT: mov z18.h, z3.h[4] ; CHECK-NEXT: strh w9, [sp, #28] ; CHECK-NEXT: fmov w9, s19 -; CHECK-NEXT: mov z19.h, z6.h[7] +; CHECK-NEXT: mov z19.h, z5.h[7] ; CHECK-NEXT: zip1 z3.h, z4.h, z3.h ; CHECK-NEXT: strh w8, [sp, #26] ; CHECK-NEXT: fmov w8, s16 ; CHECK-NEXT: mov z16.h, z4.h[4] ; CHECK-NEXT: strh w9, [sp, #24] -; CHECK-NEXT: zip1 z4.h, z6.h, z7.h +; CHECK-NEXT: zip1 z4.h, z5.h, z7.h ; CHECK-NEXT: strh w8, [sp, #22] ; CHECK-NEXT: fmov w8, s17 ; CHECK-NEXT: mov z17.h, z1.h[7] @@ -131,7 +131,7 @@ define void @zip_v32i16(ptr %a, ptr %b) { ; CHECK-NEXT: fmov w8, s18 ; CHECK-NEXT: mov z18.h, z0.h[4] ; CHECK-NEXT: zip1 z0.h, z0.h, z1.h -; CHECK-NEXT: zip1 z1.h, z2.h, z5.h +; CHECK-NEXT: zip1 z1.h, z2.h, z6.h ; CHECK-NEXT: strh w8, [sp, #54] ; CHECK-NEXT: fmov w8, s16 ; CHECK-NEXT: ldr q16, [sp, #16] @@ -143,41 +143,41 @@ define void @zip_v32i16(ptr %a, ptr %b) { ; CHECK-NEXT: mov z18.h, z7.h[7] ; CHECK-NEXT: strh w8, [sp, #48] ; CHECK-NEXT: fmov w8, s18 -; CHECK-NEXT: mov z18.h, z6.h[6] +; CHECK-NEXT: mov z18.h, z5.h[6] ; CHECK-NEXT: ldr q17, [sp, #48] ; CHECK-NEXT: strh w8, [sp, #46] ; CHECK-NEXT: fmov w8, s19 ; CHECK-NEXT: mov z19.h, z7.h[5] ; CHECK-NEXT: strh w8, [sp, #44] ; CHECK-NEXT: fmov w8, s20 -; CHECK-NEXT: mov z20.h, z6.h[5] +; CHECK-NEXT: mov z20.h, z5.h[5] ; CHECK-NEXT: strh w8, [sp, #42] ; CHECK-NEXT: fmov w8, s18 ; CHECK-NEXT: mov z18.h, z7.h[4] ; CHECK-NEXT: strh w8, [sp, #40] ; CHECK-NEXT: fmov w8, s19 -; CHECK-NEXT: mov z19.h, z6.h[4] +; CHECK-NEXT: mov z19.h, z5.h[4] ; CHECK-NEXT: strh w8, [sp, #38] ; CHECK-NEXT: fmov w8, s20 -; CHECK-NEXT: mov z20.h, z5.h[7] +; CHECK-NEXT: mov z20.h, z6.h[7] ; CHECK-NEXT: strh w8, [sp, #36] ; CHECK-NEXT: fmov w8, s18 ; CHECK-NEXT: mov z18.h, z2.h[7] ; CHECK-NEXT: strh w8, [sp, #34] ; CHECK-NEXT: fmov w8, s19 -; CHECK-NEXT: mov z19.h, z5.h[6] +; CHECK-NEXT: mov z19.h, z6.h[6] ; CHECK-NEXT: strh w8, [sp, #32] ; CHECK-NEXT: fmov w8, s20 ; CHECK-NEXT: mov z20.h, z2.h[6] ; CHECK-NEXT: strh w8, [sp, #14] ; CHECK-NEXT: fmov w8, s18 -; CHECK-NEXT: mov z18.h, z5.h[5] +; CHECK-NEXT: mov z18.h, z6.h[5] ; CHECK-NEXT: strh w8, [sp, #12] ; CHECK-NEXT: fmov w8, s19 ; CHECK-NEXT: mov z19.h, z2.h[5] ; CHECK-NEXT: strh w8, [sp, #10] ; CHECK-NEXT: fmov w8, s20 -; CHECK-NEXT: mov z20.h, z5.h[4] +; CHECK-NEXT: mov z20.h, z6.h[4] ; CHECK-NEXT: fmov w9, s19 ; CHECK-NEXT: strh w8, [sp, #8] ; CHECK-NEXT: fmov w8, s18 @@ -186,10 +186,10 @@ define void @zip_v32i16(ptr %a, ptr %b) { ; CHECK-NEXT: ldr q2, [sp, #32] ; CHECK-NEXT: strh w8, [sp, #6] ; CHECK-NEXT: fmov w8, s20 +; CHECK-NEXT: fmov w9, s18 ; CHECK-NEXT: add z2.h, z16.h, z2.h ; CHECK-NEXT: strh w8, [sp, #2] -; CHECK-NEXT: fmov w8, s18 -; CHECK-NEXT: strh w8, [sp] +; CHECK-NEXT: strh w9, [sp] ; CHECK-NEXT: ldr q4, [sp] ; CHECK-NEXT: stp q3, q2, [x0, #32] ; CHECK-NEXT: add z1.h, z17.h, z4.h @@ -471,9 +471,9 @@ define void @trn_v4f64(ptr %a, ptr %b) { define void @trn_v4f32(ptr %a, ptr %b) { ; CHECK-LABEL: trn_v4f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldr q0, [x0] ; CHECK-NEXT: ldr q1, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: trn1 z2.s, z0.s, z1.s ; CHECK-NEXT: trn2 z0.s, z0.s, z1.s ; CHECK-NEXT: fadd z0.s, p0/m, z0.s, z2.s diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ptest.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ptest.ll index f2ba4a7cc356..ab7c42b3e9e3 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ptest.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ptest.ll @@ -8,8 +8,8 @@ target triple = "aarch64-unknown-linux-gnu" define i1 @ptest_v16i1(ptr %a, ptr %b) { ; CHECK-LABEL: ptest_v16i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q0, [x0, #32] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: fcmne p1.s, p0/z, z0.s, #0.0 ; CHECK-NEXT: fcmne p2.s, p0/z, z1.s, #0.0 @@ -20,7 +20,6 @@ define i1 @ptest_v16i1(ptr %a, ptr %b) { ; CHECK-NEXT: mov z2.s, p3/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z3.s, p0/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: ptrue p0.h, vl4 -; CHECK-NEXT: ptrue p1.b, vl16 ; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: uzp1 z1.h, z1.h, z1.h ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h @@ -31,7 +30,8 @@ define i1 @ptest_v16i1(ptr %a, ptr %b) { ; CHECK-NEXT: uzp1 z0.b, z1.b, z1.b ; CHECK-NEXT: uzp1 z1.b, z3.b, z3.b ; CHECK-NEXT: splice z1.b, p0, z1.b, z0.b -; CHECK-NEXT: umaxv b0, p1, z1.b +; CHECK-NEXT: ptrue p0.b, vl16 +; CHECK-NEXT: umaxv b0, p0, z1.b ; CHECK-NEXT: fmov w8, s0 ; CHECK-NEXT: and w0, w8, #0x1 ; CHECK-NEXT: ret @@ -45,41 +45,41 @@ define i1 @ptest_v16i1(ptr %a, ptr %b) { define i1 @ptest_or_v16i1(ptr %a, ptr %b) { ; CHECK-LABEL: ptest_or_v16i1: ; CHECK: // %bb.0: +; CHECK-NEXT: ldp q1, q0, [x0, #32] ; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ldp q0, q1, [x0] -; CHECK-NEXT: ldp q2, q3, [x0, #32] -; CHECK-NEXT: ldp q4, q5, [x1] -; CHECK-NEXT: ldp q6, q7, [x1, #32] -; CHECK-NEXT: fcmne p1.s, p0/z, z3.s, #0.0 -; CHECK-NEXT: fcmne p2.s, p0/z, z2.s, #0.0 -; CHECK-NEXT: fcmne p3.s, p0/z, z1.s, #0.0 -; CHECK-NEXT: fcmne p4.s, p0/z, z0.s, #0.0 -; CHECK-NEXT: fcmne p5.s, p0/z, z7.s, #0.0 -; CHECK-NEXT: fcmne p6.s, p0/z, z6.s, #0.0 -; CHECK-NEXT: fcmne p7.s, p0/z, z5.s, #0.0 -; CHECK-NEXT: fcmne p0.s, p0/z, z4.s, #0.0 +; CHECK-NEXT: ldp q2, q3, [x0] +; CHECK-NEXT: ldp q4, q5, [x1, #32] +; CHECK-NEXT: fcmne p1.s, p0/z, z0.s, #0.0 +; CHECK-NEXT: fcmne p2.s, p0/z, z1.s, #0.0 +; CHECK-NEXT: ldp q0, q1, [x1] +; CHECK-NEXT: fcmne p3.s, p0/z, z3.s, #0.0 +; CHECK-NEXT: fcmne p4.s, p0/z, z2.s, #0.0 +; CHECK-NEXT: fcmne p5.s, p0/z, z5.s, #0.0 +; CHECK-NEXT: fcmne p6.s, p0/z, z4.s, #0.0 +; CHECK-NEXT: fcmne p7.s, p0/z, z1.s, #0.0 +; CHECK-NEXT: fcmne p0.s, p0/z, z0.s, #0.0 ; CHECK-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z1.s, p2/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z2.s, p3/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z3.s, p4/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z4.s, p5/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z5.s, p6/z, #-1 // =0xffffffffffffffff +; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: mov z6.s, p7/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z7.s, p0/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: ptrue p1.h, vl4 -; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: uzp1 z1.h, z1.h, z1.h ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h ; CHECK-NEXT: uzp1 z4.h, z4.h, z4.h ; CHECK-NEXT: uzp1 z5.h, z5.h, z5.h +; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: uzp1 z6.h, z6.h, z6.h ; CHECK-NEXT: uzp1 z7.h, z7.h, z7.h +; CHECK-NEXT: splice z1.h, p0, z1.h, z0.h +; CHECK-NEXT: splice z3.h, p0, z3.h, z2.h +; CHECK-NEXT: splice z5.h, p0, z5.h, z4.h +; CHECK-NEXT: splice z7.h, p0, z7.h, z6.h ; CHECK-NEXT: ptrue p0.b, vl8 -; CHECK-NEXT: splice z1.h, p1, z1.h, z0.h -; CHECK-NEXT: splice z3.h, p1, z3.h, z2.h -; CHECK-NEXT: splice z5.h, p1, z5.h, z4.h -; CHECK-NEXT: splice z7.h, p1, z7.h, z6.h ; CHECK-NEXT: uzp1 z0.b, z1.b, z1.b ; CHECK-NEXT: uzp1 z1.b, z3.b, z3.b ; CHECK-NEXT: uzp1 z2.b, z5.b, z5.b @@ -112,41 +112,41 @@ declare i1 @llvm.vector.reduce.or.i1.v16i1(<16 x i1>) define i1 @ptest_and_v16i1(ptr %a, ptr %b) { ; CHECK-LABEL: ptest_and_v16i1: ; CHECK: // %bb.0: +; CHECK-NEXT: ldp q1, q0, [x0, #32] ; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ldp q0, q1, [x0] -; CHECK-NEXT: ldp q2, q3, [x0, #32] -; CHECK-NEXT: ldp q4, q5, [x1] -; CHECK-NEXT: ldp q6, q7, [x1, #32] -; CHECK-NEXT: fcmne p1.s, p0/z, z3.s, #0.0 -; CHECK-NEXT: fcmne p2.s, p0/z, z2.s, #0.0 -; CHECK-NEXT: fcmne p3.s, p0/z, z1.s, #0.0 -; CHECK-NEXT: fcmne p4.s, p0/z, z0.s, #0.0 -; CHECK-NEXT: fcmne p5.s, p0/z, z7.s, #0.0 -; CHECK-NEXT: fcmne p6.s, p0/z, z6.s, #0.0 -; CHECK-NEXT: fcmne p7.s, p0/z, z5.s, #0.0 -; CHECK-NEXT: fcmne p0.s, p0/z, z4.s, #0.0 +; CHECK-NEXT: ldp q2, q3, [x0] +; CHECK-NEXT: ldp q4, q5, [x1, #32] +; CHECK-NEXT: fcmne p1.s, p0/z, z0.s, #0.0 +; CHECK-NEXT: fcmne p2.s, p0/z, z1.s, #0.0 +; CHECK-NEXT: ldp q0, q1, [x1] +; CHECK-NEXT: fcmne p3.s, p0/z, z3.s, #0.0 +; CHECK-NEXT: fcmne p4.s, p0/z, z2.s, #0.0 +; CHECK-NEXT: fcmne p5.s, p0/z, z5.s, #0.0 +; CHECK-NEXT: fcmne p6.s, p0/z, z4.s, #0.0 +; CHECK-NEXT: fcmne p7.s, p0/z, z1.s, #0.0 +; CHECK-NEXT: fcmne p0.s, p0/z, z0.s, #0.0 ; CHECK-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z1.s, p2/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z2.s, p3/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z3.s, p4/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z4.s, p5/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z5.s, p6/z, #-1 // =0xffffffffffffffff +; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: mov z6.s, p7/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z7.s, p0/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: ptrue p1.h, vl4 -; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: uzp1 z1.h, z1.h, z1.h ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h ; CHECK-NEXT: uzp1 z4.h, z4.h, z4.h ; CHECK-NEXT: uzp1 z5.h, z5.h, z5.h +; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: uzp1 z6.h, z6.h, z6.h ; CHECK-NEXT: uzp1 z7.h, z7.h, z7.h +; CHECK-NEXT: splice z1.h, p0, z1.h, z0.h +; CHECK-NEXT: splice z3.h, p0, z3.h, z2.h +; CHECK-NEXT: splice z5.h, p0, z5.h, z4.h +; CHECK-NEXT: splice z7.h, p0, z7.h, z6.h ; CHECK-NEXT: ptrue p0.b, vl8 -; CHECK-NEXT: splice z1.h, p1, z1.h, z0.h -; CHECK-NEXT: splice z3.h, p1, z3.h, z2.h -; CHECK-NEXT: splice z5.h, p1, z5.h, z4.h -; CHECK-NEXT: splice z7.h, p1, z7.h, z6.h ; CHECK-NEXT: uzp1 z0.b, z1.b, z1.b ; CHECK-NEXT: uzp1 z1.b, z3.b, z3.b ; CHECK-NEXT: uzp1 z2.b, z5.b, z5.b diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-rev.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-rev.ll index f686efff67b6..bfa931044bc5 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-rev.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-rev.ll @@ -49,8 +49,8 @@ define <16 x i8> @bitreverse_v16i8(<16 x i8> %op) { define void @bitreverse_v32i8(ptr %a) { ; CHECK-LABEL: bitreverse_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: rbit z0.b, p0/m, z0.b ; CHECK-NEXT: rbit z1.b, p0/m, z1.b ; CHECK-NEXT: stp q0, q1, [x0] @@ -101,8 +101,8 @@ define <8 x i16> @bitreverse_v8i16(<8 x i16> %op) { define void @bitreverse_v16i16(ptr %a) { ; CHECK-LABEL: bitreverse_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: rbit z0.h, p0/m, z0.h ; CHECK-NEXT: rbit z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x0] @@ -140,8 +140,8 @@ define <4 x i32> @bitreverse_v4i32(<4 x i32> %op) { define void @bitreverse_v8i32(ptr %a) { ; CHECK-LABEL: bitreverse_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: rbit z0.s, p0/m, z0.s ; CHECK-NEXT: rbit z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -179,8 +179,8 @@ define <2 x i64> @bitreverse_v2i64(<2 x i64> %op) { define void @bitreverse_v4i64(ptr %a) { ; CHECK-LABEL: bitreverse_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: rbit z0.d, p0/m, z0.d ; CHECK-NEXT: rbit z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] @@ -235,8 +235,8 @@ define <8 x i16> @bswap_v8i16(<8 x i16> %op) { define void @bswap_v16i16(ptr %a) { ; CHECK-LABEL: bswap_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: revb z0.h, p0/m, z0.h ; CHECK-NEXT: revb z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x0] @@ -274,8 +274,8 @@ define <4 x i32> @bswap_v4i32(<4 x i32> %op) { define void @bswap_v8i32(ptr %a) { ; CHECK-LABEL: bswap_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: revb z0.s, p0/m, z0.s ; CHECK-NEXT: revb z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -313,8 +313,8 @@ define <2 x i64> @bswap_v2i64(<2 x i64> %op) { define void @bswap_v4i64(ptr %a) { ; CHECK-LABEL: bswap_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: revb z0.d, p0/m, z0.d ; CHECK-NEXT: revb z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-sdiv-pow2.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-sdiv-pow2.ll index 76bb465774d5..9dd42e7831e0 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-sdiv-pow2.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-sdiv-pow2.ll @@ -45,8 +45,8 @@ define <16 x i8> @sdiv_v16i8(<16 x i8> %op1) { define void @sdiv_v32i8(ptr %a) { ; CHECK-LABEL: sdiv_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: asrd z0.b, p0/m, z0.b, #5 ; CHECK-NEXT: asrd z1.b, p0/m, z1.b, #5 ; CHECK-NEXT: stp q0, q1, [x0] @@ -97,8 +97,8 @@ define <8 x i16> @sdiv_v8i16(<8 x i16> %op1) { define void @sdiv_v16i16(ptr %a) { ; CHECK-LABEL: sdiv_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: asrd z0.h, p0/m, z0.h, #5 ; CHECK-NEXT: asrd z1.h, p0/m, z1.h, #5 ; CHECK-NEXT: stp q0, q1, [x0] @@ -136,8 +136,8 @@ define <4 x i32> @sdiv_v4i32(<4 x i32> %op1) { define void @sdiv_v8i32(ptr %a) { ; CHECK-LABEL: sdiv_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: asrd z0.s, p0/m, z0.s, #5 ; CHECK-NEXT: asrd z1.s, p0/m, z1.s, #5 ; CHECK-NEXT: stp q0, q1, [x0] @@ -176,8 +176,8 @@ define <2 x i64> @sdiv_v2i64(<2 x i64> %op1) { define void @sdiv_v4i64(ptr %a) { ; CHECK-LABEL: sdiv_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: asrd z0.d, p0/m, z0.d, #5 ; CHECK-NEXT: asrd z1.d, p0/m, z1.d, #5 ; CHECK-NEXT: stp q0, q1, [x0] diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-shuffle.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-shuffle.ll index ff1f8699b91a..ad0d4ef0afef 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-shuffle.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-shuffle.ll @@ -37,9 +37,9 @@ define void @interleave_store_without_splat(ptr %a, <4 x i32> %v1, <4 x i32> %v2 define void @interleave_store_legalization(ptr %a, <8 x i32> %v1, <8 x i32> %v2) { ; CHECK-LABEL: interleave_store_legalization: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: mov z5.d, z2.d ; CHECK-NEXT: // kill: def $q3 killed $q3 def $z2_z3 +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: mov x8, #8 // =0x8 ; CHECK-NEXT: mov z4.d, z0.d ; CHECK-NEXT: mov z2.d, z1.d diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-stores.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-stores.ll index 367ccbeeea81..06709ca3685c 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-stores.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-stores.ll @@ -8,8 +8,8 @@ target triple = "aarch64-unknown-linux-gnu" define void @store_v4i8(ptr %a) { ; CHECK-LABEL: store_v4i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: mov z0.h, #0 // =0x0 +; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: st1b { z0.h }, p0, [x0] ; CHECK-NEXT: ret store <4 x i8> zeroinitializer, ptr %a @@ -49,8 +49,8 @@ define void @store_v32i8(ptr %a) { define void @store_v2i16(ptr %a) { ; CHECK-LABEL: store_v2i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: mov z0.s, #0 // =0x0 +; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: st1h { z0.s }, p0, [x0] ; CHECK-NEXT: ret store <2 x i16> zeroinitializer, ptr %a diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc.ll index 4fef67831401..70219dd30f76 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc.ll @@ -87,51 +87,51 @@ define void @trunc_v64i16_v64i8(ptr %in, ptr %out) nounwind { define void @trunc_v128i16_v128i8(ptr %in, ptr %out) nounwind { ; CHECK-LABEL: trunc_v128i16_v128i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ldp q0, q1, [x0, #192] +; CHECK-NEXT: ldp q2, q3, [x0, #192] ; CHECK-NEXT: ptrue p0.b, vl8 -; CHECK-NEXT: ldp q6, q7, [x0, #224] -; CHECK-NEXT: ldp q2, q3, [x0, #32] -; CHECK-NEXT: uzp1 z1.b, z1.b, z1.b -; CHECK-NEXT: uzp1 z0.b, z0.b, z0.b -; CHECK-NEXT: uzp1 z7.b, z7.b, z7.b -; CHECK-NEXT: uzp1 z6.b, z6.b, z6.b -; CHECK-NEXT: ldp q4, q5, [x0] +; CHECK-NEXT: ldp q6, q7, [x0, #64] +; CHECK-NEXT: ldp q16, q17, [x0, #224] ; CHECK-NEXT: uzp1 z3.b, z3.b, z3.b -; CHECK-NEXT: ldp q16, q17, [x0, #64] ; CHECK-NEXT: uzp1 z2.b, z2.b, z2.b -; CHECK-NEXT: ldp q18, q19, [x0, #128] -; CHECK-NEXT: splice z0.b, p0, z0.b, z1.b ; CHECK-NEXT: ldp q20, q21, [x0, #160] -; CHECK-NEXT: splice z6.b, p0, z6.b, z7.b -; CHECK-NEXT: ldp q22, q23, [x0, #96] -; CHECK-NEXT: uzp1 z1.b, z17.b, z17.b -; CHECK-NEXT: uzp1 z19.b, z19.b, z19.b -; CHECK-NEXT: uzp1 z18.b, z18.b, z18.b +; CHECK-NEXT: uzp1 z7.b, z7.b, z7.b +; CHECK-NEXT: ldp q0, q1, [x0, #32] +; CHECK-NEXT: uzp1 z17.b, z17.b, z17.b +; CHECK-NEXT: ldp q4, q5, [x0, #96] ; CHECK-NEXT: uzp1 z16.b, z16.b, z16.b -; CHECK-NEXT: uzp1 z21.b, z21.b, z21.b +; CHECK-NEXT: ldp q18, q19, [x0, #128] +; CHECK-NEXT: splice z2.b, p0, z2.b, z3.b +; CHECK-NEXT: uzp1 z3.b, z21.b, z21.b ; CHECK-NEXT: uzp1 z20.b, z20.b, z20.b -; CHECK-NEXT: uzp1 z5.b, z5.b, z5.b -; CHECK-NEXT: uzp1 z7.b, z23.b, z23.b -; CHECK-NEXT: uzp1 z17.b, z22.b, z22.b +; CHECK-NEXT: uzp1 z6.b, z6.b, z6.b +; CHECK-NEXT: ldp q21, q22, [x0] +; CHECK-NEXT: splice z16.b, p0, z16.b, z17.b +; CHECK-NEXT: uzp1 z19.b, z19.b, z19.b +; CHECK-NEXT: uzp1 z18.b, z18.b, z18.b ; CHECK-NEXT: uzp1 z4.b, z4.b, z4.b -; CHECK-NEXT: splice z2.b, p0, z2.b, z3.b -; CHECK-NEXT: add z0.b, z0.b, z0.b +; CHECK-NEXT: splice z20.b, p0, z20.b, z3.b +; CHECK-NEXT: uzp1 z3.b, z5.b, z5.b +; CHECK-NEXT: splice z6.b, p0, z6.b, z7.b +; CHECK-NEXT: uzp1 z5.b, z22.b, z22.b +; CHECK-NEXT: uzp1 z7.b, z21.b, z21.b +; CHECK-NEXT: uzp1 z1.b, z1.b, z1.b +; CHECK-NEXT: uzp1 z0.b, z0.b, z0.b ; CHECK-NEXT: splice z18.b, p0, z18.b, z19.b -; CHECK-NEXT: splice z16.b, p0, z16.b, z1.b -; CHECK-NEXT: add z1.b, z6.b, z6.b -; CHECK-NEXT: splice z20.b, p0, z20.b, z21.b -; CHECK-NEXT: splice z17.b, p0, z17.b, z7.b -; CHECK-NEXT: splice z4.b, p0, z4.b, z5.b -; CHECK-NEXT: stp q0, q1, [x1, #96] ; CHECK-NEXT: add z2.b, z2.b, z2.b +; CHECK-NEXT: splice z4.b, p0, z4.b, z3.b +; CHECK-NEXT: add z3.b, z16.b, z16.b +; CHECK-NEXT: splice z7.b, p0, z7.b, z5.b +; CHECK-NEXT: splice z0.b, p0, z0.b, z1.b +; CHECK-NEXT: add z1.b, z20.b, z20.b ; CHECK-NEXT: add z5.b, z18.b, z18.b -; CHECK-NEXT: add z0.b, z16.b, z16.b -; CHECK-NEXT: add z3.b, z20.b, z20.b -; CHECK-NEXT: add z1.b, z17.b, z17.b -; CHECK-NEXT: add z4.b, z4.b, z4.b -; CHECK-NEXT: stp q5, q3, [x1, #64] -; CHECK-NEXT: stp q4, q2, [x1] -; CHECK-NEXT: stp q0, q1, [x1, #32] +; CHECK-NEXT: stp q2, q3, [x1, #96] +; CHECK-NEXT: add z2.b, z6.b, z6.b +; CHECK-NEXT: add z3.b, z4.b, z4.b +; CHECK-NEXT: add z4.b, z7.b, z7.b +; CHECK-NEXT: add z0.b, z0.b, z0.b +; CHECK-NEXT: stp q5, q1, [x1, #64] +; CHECK-NEXT: stp q2, q3, [x1, #32] +; CHECK-NEXT: stp q4, q0, [x1] ; CHECK-NEXT: ret %a = load <128 x i16>, ptr %in %b = trunc <128 x i16> %a to <128 x i8> @@ -226,55 +226,55 @@ define void @trunc_v32i32_v32i8(ptr %in, ptr %out) nounwind { define void @trunc_v64i32_v64i8(ptr %in, ptr %out) nounwind { ; CHECK-LABEL: trunc_v64i32_v64i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ldp q0, q1, [x0, #64] -; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: ldp q2, q3, [x0, #160] -; CHECK-NEXT: ptrue p1.b, vl8 -; CHECK-NEXT: ldp q4, q5, [x0, #96] -; CHECK-NEXT: ldp q6, q7, [x0] -; CHECK-NEXT: uzp1 z1.h, z1.h, z1.h -; CHECK-NEXT: ldp q16, q17, [x0, #128] +; CHECK-NEXT: ptrue p0.h, vl4 +; CHECK-NEXT: ldp q4, q5, [x0, #128] +; CHECK-NEXT: ldp q0, q1, [x0, #64] +; CHECK-NEXT: ldp q6, q7, [x0, #96] ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h -; CHECK-NEXT: ldp q18, q19, [x0, #192] ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h -; CHECK-NEXT: ldp q20, q21, [x0, #224] +; CHECK-NEXT: uzp1 z5.h, z5.h, z5.h +; CHECK-NEXT: uzp1 z4.h, z4.h, z4.h +; CHECK-NEXT: ldp q16, q17, [x0] +; CHECK-NEXT: uzp1 z1.h, z1.h, z1.h +; CHECK-NEXT: ldp q18, q19, [x0, #192] ; CHECK-NEXT: uzp1 z7.h, z7.h, z7.h +; CHECK-NEXT: ldp q20, q21, [x0, #224] +; CHECK-NEXT: splice z2.h, p0, z2.h, z3.h ; CHECK-NEXT: ldp q22, q23, [x0, #32] -; CHECK-NEXT: uzp1 z17.h, z17.h, z17.h -; CHECK-NEXT: uzp1 z16.h, z16.h, z16.h +; CHECK-NEXT: splice z4.h, p0, z4.h, z5.h ; CHECK-NEXT: uzp1 z19.h, z19.h, z19.h ; CHECK-NEXT: uzp1 z18.h, z18.h, z18.h -; CHECK-NEXT: uzp1 z21.h, z21.h, z21.h -; CHECK-NEXT: uzp1 z20.h, z20.h, z20.h +; CHECK-NEXT: uzp1 z17.h, z17.h, z17.h +; CHECK-NEXT: uzp1 z3.h, z21.h, z21.h +; CHECK-NEXT: uzp1 z5.h, z20.h, z20.h +; CHECK-NEXT: uzp1 z16.h, z16.h, z16.h +; CHECK-NEXT: uzp1 z20.h, z23.h, z23.h +; CHECK-NEXT: uzp1 z21.h, z22.h, z22.h ; CHECK-NEXT: uzp1 z6.h, z6.h, z6.h -; CHECK-NEXT: uzp1 z23.h, z23.h, z23.h -; CHECK-NEXT: uzp1 z22.h, z22.h, z22.h -; CHECK-NEXT: uzp1 z5.h, z5.h, z5.h -; CHECK-NEXT: uzp1 z4.h, z4.h, z4.h ; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h -; CHECK-NEXT: splice z2.h, p0, z2.h, z3.h -; CHECK-NEXT: splice z16.h, p0, z16.h, z17.h ; CHECK-NEXT: splice z18.h, p0, z18.h, z19.h -; CHECK-NEXT: splice z20.h, p0, z20.h, z21.h +; CHECK-NEXT: splice z5.h, p0, z5.h, z3.h +; CHECK-NEXT: splice z16.h, p0, z16.h, z17.h +; CHECK-NEXT: splice z21.h, p0, z21.h, z20.h ; CHECK-NEXT: splice z6.h, p0, z6.h, z7.h -; CHECK-NEXT: splice z22.h, p0, z22.h, z23.h -; CHECK-NEXT: splice z4.h, p0, z4.h, z5.h ; CHECK-NEXT: splice z0.h, p0, z0.h, z1.h ; CHECK-NEXT: uzp1 z1.b, z2.b, z2.b -; CHECK-NEXT: uzp1 z2.b, z16.b, z16.b -; CHECK-NEXT: uzp1 z5.b, z18.b, z18.b -; CHECK-NEXT: uzp1 z3.b, z20.b, z20.b -; CHECK-NEXT: uzp1 z6.b, z6.b, z6.b -; CHECK-NEXT: uzp1 z7.b, z22.b, z22.b -; CHECK-NEXT: uzp1 z4.b, z4.b, z4.b +; CHECK-NEXT: uzp1 z2.b, z4.b, z4.b +; CHECK-NEXT: ptrue p0.b, vl8 +; CHECK-NEXT: uzp1 z4.b, z18.b, z18.b +; CHECK-NEXT: uzp1 z3.b, z5.b, z5.b +; CHECK-NEXT: uzp1 z7.b, z16.b, z16.b +; CHECK-NEXT: uzp1 z5.b, z21.b, z21.b +; CHECK-NEXT: splice z2.b, p0, z2.b, z1.b +; CHECK-NEXT: uzp1 z1.b, z6.b, z6.b ; CHECK-NEXT: uzp1 z0.b, z0.b, z0.b -; CHECK-NEXT: splice z2.b, p1, z2.b, z1.b -; CHECK-NEXT: splice z5.b, p1, z5.b, z3.b -; CHECK-NEXT: splice z6.b, p1, z6.b, z7.b -; CHECK-NEXT: splice z0.b, p1, z0.b, z4.b +; CHECK-NEXT: splice z4.b, p0, z4.b, z3.b +; CHECK-NEXT: splice z7.b, p0, z7.b, z5.b +; CHECK-NEXT: splice z0.b, p0, z0.b, z1.b ; CHECK-NEXT: add z1.b, z2.b, z2.b -; CHECK-NEXT: add z2.b, z5.b, z5.b -; CHECK-NEXT: add z3.b, z6.b, z6.b +; CHECK-NEXT: add z2.b, z4.b, z4.b +; CHECK-NEXT: add z3.b, z7.b, z7.b ; CHECK-NEXT: add z0.b, z0.b, z0.b ; CHECK-NEXT: stp q1, q2, [x1, #32] ; CHECK-NEXT: stp q3, q0, [x1] @@ -368,51 +368,51 @@ define void @trunc_v32i32_v32i16(ptr %in, ptr %out) nounwind { define void @trunc_v64i32_v64i16(ptr %in, ptr %out) nounwind { ; CHECK-LABEL: trunc_v64i32_v64i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ldp q0, q1, [x0, #192] +; CHECK-NEXT: ldp q2, q3, [x0, #192] ; CHECK-NEXT: ptrue p0.h, vl4 -; CHECK-NEXT: ldp q6, q7, [x0, #224] -; CHECK-NEXT: ldp q2, q3, [x0, #32] -; CHECK-NEXT: uzp1 z1.h, z1.h, z1.h -; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h -; CHECK-NEXT: uzp1 z7.h, z7.h, z7.h -; CHECK-NEXT: uzp1 z6.h, z6.h, z6.h -; CHECK-NEXT: ldp q4, q5, [x0] +; CHECK-NEXT: ldp q6, q7, [x0, #64] +; CHECK-NEXT: ldp q16, q17, [x0, #224] ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h -; CHECK-NEXT: ldp q16, q17, [x0, #64] ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h -; CHECK-NEXT: ldp q18, q19, [x0, #128] -; CHECK-NEXT: splice z0.h, p0, z0.h, z1.h ; CHECK-NEXT: ldp q20, q21, [x0, #160] -; CHECK-NEXT: splice z6.h, p0, z6.h, z7.h -; CHECK-NEXT: ldp q22, q23, [x0, #96] -; CHECK-NEXT: uzp1 z1.h, z17.h, z17.h -; CHECK-NEXT: uzp1 z19.h, z19.h, z19.h -; CHECK-NEXT: uzp1 z18.h, z18.h, z18.h +; CHECK-NEXT: uzp1 z7.h, z7.h, z7.h +; CHECK-NEXT: ldp q0, q1, [x0, #32] +; CHECK-NEXT: uzp1 z17.h, z17.h, z17.h +; CHECK-NEXT: ldp q4, q5, [x0, #96] ; CHECK-NEXT: uzp1 z16.h, z16.h, z16.h -; CHECK-NEXT: uzp1 z21.h, z21.h, z21.h +; CHECK-NEXT: ldp q18, q19, [x0, #128] +; CHECK-NEXT: splice z2.h, p0, z2.h, z3.h +; CHECK-NEXT: uzp1 z3.h, z21.h, z21.h ; CHECK-NEXT: uzp1 z20.h, z20.h, z20.h -; CHECK-NEXT: uzp1 z5.h, z5.h, z5.h -; CHECK-NEXT: uzp1 z7.h, z23.h, z23.h -; CHECK-NEXT: uzp1 z17.h, z22.h, z22.h +; CHECK-NEXT: uzp1 z6.h, z6.h, z6.h +; CHECK-NEXT: ldp q21, q22, [x0] +; CHECK-NEXT: splice z16.h, p0, z16.h, z17.h +; CHECK-NEXT: uzp1 z19.h, z19.h, z19.h +; CHECK-NEXT: uzp1 z18.h, z18.h, z18.h ; CHECK-NEXT: uzp1 z4.h, z4.h, z4.h -; CHECK-NEXT: splice z2.h, p0, z2.h, z3.h -; CHECK-NEXT: add z0.h, z0.h, z0.h +; CHECK-NEXT: splice z20.h, p0, z20.h, z3.h +; CHECK-NEXT: uzp1 z3.h, z5.h, z5.h +; CHECK-NEXT: splice z6.h, p0, z6.h, z7.h +; CHECK-NEXT: uzp1 z5.h, z22.h, z22.h +; CHECK-NEXT: uzp1 z7.h, z21.h, z21.h +; CHECK-NEXT: uzp1 z1.h, z1.h, z1.h +; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: splice z18.h, p0, z18.h, z19.h -; CHECK-NEXT: splice z16.h, p0, z16.h, z1.h -; CHECK-NEXT: add z1.h, z6.h, z6.h -; CHECK-NEXT: splice z20.h, p0, z20.h, z21.h -; CHECK-NEXT: splice z17.h, p0, z17.h, z7.h -; CHECK-NEXT: splice z4.h, p0, z4.h, z5.h -; CHECK-NEXT: stp q0, q1, [x1, #96] ; CHECK-NEXT: add z2.h, z2.h, z2.h +; CHECK-NEXT: splice z4.h, p0, z4.h, z3.h +; CHECK-NEXT: add z3.h, z16.h, z16.h +; CHECK-NEXT: splice z7.h, p0, z7.h, z5.h +; CHECK-NEXT: splice z0.h, p0, z0.h, z1.h +; CHECK-NEXT: add z1.h, z20.h, z20.h ; CHECK-NEXT: add z5.h, z18.h, z18.h -; CHECK-NEXT: add z0.h, z16.h, z16.h -; CHECK-NEXT: add z3.h, z20.h, z20.h -; CHECK-NEXT: add z1.h, z17.h, z17.h -; CHECK-NEXT: add z4.h, z4.h, z4.h -; CHECK-NEXT: stp q5, q3, [x1, #64] -; CHECK-NEXT: stp q4, q2, [x1] -; CHECK-NEXT: stp q0, q1, [x1, #32] +; CHECK-NEXT: stp q2, q3, [x1, #96] +; CHECK-NEXT: add z2.h, z6.h, z6.h +; CHECK-NEXT: add z3.h, z4.h, z4.h +; CHECK-NEXT: add z4.h, z7.h, z7.h +; CHECK-NEXT: add z0.h, z0.h, z0.h +; CHECK-NEXT: stp q5, q1, [x1, #64] +; CHECK-NEXT: stp q2, q3, [x1, #32] +; CHECK-NEXT: stp q4, q0, [x1] ; CHECK-NEXT: ret %a = load <64 x i32>, ptr %in %b = trunc <64 x i32> %a to <64 x i16> @@ -535,19 +535,19 @@ define void @trunc_v32i64_v32i8(ptr %in, ptr %out) nounwind { ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s ; CHECK-NEXT: splice z2.s, p0, z2.s, z3.s ; CHECK-NEXT: splice z16.s, p0, z16.s, z17.s -; CHECK-NEXT: splice z18.s, p0, z18.s, z19.s ; CHECK-NEXT: splice z20.s, p0, z20.s, z21.s -; CHECK-NEXT: splice z6.s, p0, z6.s, z7.s +; CHECK-NEXT: splice z18.s, p0, z18.s, z19.s ; CHECK-NEXT: splice z22.s, p0, z22.s, z23.s +; CHECK-NEXT: splice z6.s, p0, z6.s, z7.s ; CHECK-NEXT: splice z4.s, p0, z4.s, z5.s ; CHECK-NEXT: splice z0.s, p0, z0.s, z1.s ; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: uzp1 z1.h, z2.h, z2.h ; CHECK-NEXT: uzp1 z2.h, z16.h, z16.h -; CHECK-NEXT: uzp1 z5.h, z18.h, z18.h ; CHECK-NEXT: uzp1 z3.h, z20.h, z20.h -; CHECK-NEXT: uzp1 z6.h, z6.h, z6.h +; CHECK-NEXT: uzp1 z5.h, z18.h, z18.h ; CHECK-NEXT: uzp1 z7.h, z22.h, z22.h +; CHECK-NEXT: uzp1 z6.h, z6.h, z6.h ; CHECK-NEXT: uzp1 z4.h, z4.h, z4.h ; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: splice z2.h, p0, z2.h, z1.h @@ -658,55 +658,55 @@ define void @trunc_v16i64_v16i16(ptr %in, ptr %out) nounwind { define void @trunc_v32i64_v32i16(ptr %in, ptr %out) nounwind { ; CHECK-LABEL: trunc_v32i64_v32i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ldp q0, q1, [x0, #64] -; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: ldp q2, q3, [x0, #160] -; CHECK-NEXT: ptrue p1.h, vl4 -; CHECK-NEXT: ldp q4, q5, [x0, #96] -; CHECK-NEXT: ldp q6, q7, [x0] -; CHECK-NEXT: uzp1 z1.s, z1.s, z1.s -; CHECK-NEXT: ldp q16, q17, [x0, #128] +; CHECK-NEXT: ptrue p0.s, vl2 +; CHECK-NEXT: ldp q4, q5, [x0, #128] +; CHECK-NEXT: ldp q0, q1, [x0, #64] +; CHECK-NEXT: ldp q6, q7, [x0, #96] ; CHECK-NEXT: uzp1 z3.s, z3.s, z3.s -; CHECK-NEXT: ldp q18, q19, [x0, #192] ; CHECK-NEXT: uzp1 z2.s, z2.s, z2.s -; CHECK-NEXT: ldp q20, q21, [x0, #224] +; CHECK-NEXT: uzp1 z5.s, z5.s, z5.s +; CHECK-NEXT: uzp1 z4.s, z4.s, z4.s +; CHECK-NEXT: ldp q16, q17, [x0] +; CHECK-NEXT: uzp1 z1.s, z1.s, z1.s +; CHECK-NEXT: ldp q18, q19, [x0, #192] ; CHECK-NEXT: uzp1 z7.s, z7.s, z7.s +; CHECK-NEXT: ldp q20, q21, [x0, #224] +; CHECK-NEXT: splice z2.s, p0, z2.s, z3.s ; CHECK-NEXT: ldp q22, q23, [x0, #32] -; CHECK-NEXT: uzp1 z17.s, z17.s, z17.s -; CHECK-NEXT: uzp1 z16.s, z16.s, z16.s +; CHECK-NEXT: splice z4.s, p0, z4.s, z5.s ; CHECK-NEXT: uzp1 z19.s, z19.s, z19.s ; CHECK-NEXT: uzp1 z18.s, z18.s, z18.s -; CHECK-NEXT: uzp1 z21.s, z21.s, z21.s -; CHECK-NEXT: uzp1 z20.s, z20.s, z20.s +; CHECK-NEXT: uzp1 z17.s, z17.s, z17.s +; CHECK-NEXT: uzp1 z3.s, z21.s, z21.s +; CHECK-NEXT: uzp1 z5.s, z20.s, z20.s +; CHECK-NEXT: uzp1 z16.s, z16.s, z16.s +; CHECK-NEXT: uzp1 z20.s, z23.s, z23.s +; CHECK-NEXT: uzp1 z21.s, z22.s, z22.s ; CHECK-NEXT: uzp1 z6.s, z6.s, z6.s -; CHECK-NEXT: uzp1 z23.s, z23.s, z23.s -; CHECK-NEXT: uzp1 z22.s, z22.s, z22.s -; CHECK-NEXT: uzp1 z5.s, z5.s, z5.s -; CHECK-NEXT: uzp1 z4.s, z4.s, z4.s ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s -; CHECK-NEXT: splice z2.s, p0, z2.s, z3.s -; CHECK-NEXT: splice z16.s, p0, z16.s, z17.s ; CHECK-NEXT: splice z18.s, p0, z18.s, z19.s -; CHECK-NEXT: splice z20.s, p0, z20.s, z21.s +; CHECK-NEXT: splice z5.s, p0, z5.s, z3.s +; CHECK-NEXT: splice z16.s, p0, z16.s, z17.s +; CHECK-NEXT: splice z21.s, p0, z21.s, z20.s ; CHECK-NEXT: splice z6.s, p0, z6.s, z7.s -; CHECK-NEXT: splice z22.s, p0, z22.s, z23.s -; CHECK-NEXT: splice z4.s, p0, z4.s, z5.s ; CHECK-NEXT: splice z0.s, p0, z0.s, z1.s ; CHECK-NEXT: uzp1 z1.h, z2.h, z2.h -; CHECK-NEXT: uzp1 z2.h, z16.h, z16.h -; CHECK-NEXT: uzp1 z5.h, z18.h, z18.h -; CHECK-NEXT: uzp1 z3.h, z20.h, z20.h -; CHECK-NEXT: uzp1 z6.h, z6.h, z6.h -; CHECK-NEXT: uzp1 z7.h, z22.h, z22.h -; CHECK-NEXT: uzp1 z4.h, z4.h, z4.h +; CHECK-NEXT: uzp1 z2.h, z4.h, z4.h +; CHECK-NEXT: ptrue p0.h, vl4 +; CHECK-NEXT: uzp1 z4.h, z18.h, z18.h +; CHECK-NEXT: uzp1 z3.h, z5.h, z5.h +; CHECK-NEXT: uzp1 z7.h, z16.h, z16.h +; CHECK-NEXT: uzp1 z5.h, z21.h, z21.h +; CHECK-NEXT: splice z2.h, p0, z2.h, z1.h +; CHECK-NEXT: uzp1 z1.h, z6.h, z6.h ; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h -; CHECK-NEXT: splice z2.h, p1, z2.h, z1.h -; CHECK-NEXT: splice z5.h, p1, z5.h, z3.h -; CHECK-NEXT: splice z6.h, p1, z6.h, z7.h -; CHECK-NEXT: splice z0.h, p1, z0.h, z4.h +; CHECK-NEXT: splice z4.h, p0, z4.h, z3.h +; CHECK-NEXT: splice z7.h, p0, z7.h, z5.h +; CHECK-NEXT: splice z0.h, p0, z0.h, z1.h ; CHECK-NEXT: add z1.h, z2.h, z2.h -; CHECK-NEXT: add z2.h, z5.h, z5.h -; CHECK-NEXT: add z3.h, z6.h, z6.h +; CHECK-NEXT: add z2.h, z4.h, z4.h +; CHECK-NEXT: add z3.h, z7.h, z7.h ; CHECK-NEXT: add z0.h, z0.h, z0.h ; CHECK-NEXT: stp q1, q2, [x1, #32] ; CHECK-NEXT: stp q3, q0, [x1] @@ -800,51 +800,51 @@ define void @trunc_v16i64_v16i32(ptr %in, ptr %out) nounwind { define void @trunc_v32i64_v32i32(ptr %in, ptr %out) nounwind { ; CHECK-LABEL: trunc_v32i64_v32i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ldp q0, q1, [x0, #192] +; CHECK-NEXT: ldp q2, q3, [x0, #192] ; CHECK-NEXT: ptrue p0.s, vl2 -; CHECK-NEXT: ldp q6, q7, [x0, #224] -; CHECK-NEXT: ldp q2, q3, [x0, #32] -; CHECK-NEXT: uzp1 z1.s, z1.s, z1.s -; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s -; CHECK-NEXT: uzp1 z7.s, z7.s, z7.s -; CHECK-NEXT: uzp1 z6.s, z6.s, z6.s -; CHECK-NEXT: ldp q4, q5, [x0] +; CHECK-NEXT: ldp q6, q7, [x0, #64] +; CHECK-NEXT: ldp q16, q17, [x0, #224] ; CHECK-NEXT: uzp1 z3.s, z3.s, z3.s -; CHECK-NEXT: ldp q16, q17, [x0, #64] ; CHECK-NEXT: uzp1 z2.s, z2.s, z2.s -; CHECK-NEXT: ldp q18, q19, [x0, #128] -; CHECK-NEXT: splice z0.s, p0, z0.s, z1.s ; CHECK-NEXT: ldp q20, q21, [x0, #160] -; CHECK-NEXT: splice z6.s, p0, z6.s, z7.s -; CHECK-NEXT: ldp q22, q23, [x0, #96] -; CHECK-NEXT: uzp1 z1.s, z17.s, z17.s -; CHECK-NEXT: uzp1 z19.s, z19.s, z19.s -; CHECK-NEXT: uzp1 z18.s, z18.s, z18.s +; CHECK-NEXT: uzp1 z7.s, z7.s, z7.s +; CHECK-NEXT: ldp q0, q1, [x0, #32] +; CHECK-NEXT: uzp1 z17.s, z17.s, z17.s +; CHECK-NEXT: ldp q4, q5, [x0, #96] ; CHECK-NEXT: uzp1 z16.s, z16.s, z16.s -; CHECK-NEXT: uzp1 z21.s, z21.s, z21.s +; CHECK-NEXT: ldp q18, q19, [x0, #128] +; CHECK-NEXT: splice z2.s, p0, z2.s, z3.s +; CHECK-NEXT: uzp1 z3.s, z21.s, z21.s ; CHECK-NEXT: uzp1 z20.s, z20.s, z20.s -; CHECK-NEXT: uzp1 z5.s, z5.s, z5.s -; CHECK-NEXT: uzp1 z7.s, z23.s, z23.s -; CHECK-NEXT: uzp1 z17.s, z22.s, z22.s +; CHECK-NEXT: uzp1 z6.s, z6.s, z6.s +; CHECK-NEXT: ldp q21, q22, [x0] +; CHECK-NEXT: splice z16.s, p0, z16.s, z17.s +; CHECK-NEXT: uzp1 z19.s, z19.s, z19.s +; CHECK-NEXT: uzp1 z18.s, z18.s, z18.s ; CHECK-NEXT: uzp1 z4.s, z4.s, z4.s -; CHECK-NEXT: splice z2.s, p0, z2.s, z3.s -; CHECK-NEXT: add z0.s, z0.s, z0.s +; CHECK-NEXT: splice z20.s, p0, z20.s, z3.s +; CHECK-NEXT: uzp1 z3.s, z5.s, z5.s +; CHECK-NEXT: splice z6.s, p0, z6.s, z7.s +; CHECK-NEXT: uzp1 z5.s, z22.s, z22.s +; CHECK-NEXT: uzp1 z7.s, z21.s, z21.s +; CHECK-NEXT: uzp1 z1.s, z1.s, z1.s +; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s ; CHECK-NEXT: splice z18.s, p0, z18.s, z19.s -; CHECK-NEXT: splice z16.s, p0, z16.s, z1.s -; CHECK-NEXT: add z1.s, z6.s, z6.s -; CHECK-NEXT: splice z20.s, p0, z20.s, z21.s -; CHECK-NEXT: splice z17.s, p0, z17.s, z7.s -; CHECK-NEXT: splice z4.s, p0, z4.s, z5.s -; CHECK-NEXT: stp q0, q1, [x1, #96] ; CHECK-NEXT: add z2.s, z2.s, z2.s +; CHECK-NEXT: splice z4.s, p0, z4.s, z3.s +; CHECK-NEXT: add z3.s, z16.s, z16.s +; CHECK-NEXT: splice z7.s, p0, z7.s, z5.s +; CHECK-NEXT: splice z0.s, p0, z0.s, z1.s +; CHECK-NEXT: add z1.s, z20.s, z20.s ; CHECK-NEXT: add z5.s, z18.s, z18.s -; CHECK-NEXT: add z0.s, z16.s, z16.s -; CHECK-NEXT: add z3.s, z20.s, z20.s -; CHECK-NEXT: add z1.s, z17.s, z17.s -; CHECK-NEXT: add z4.s, z4.s, z4.s -; CHECK-NEXT: stp q5, q3, [x1, #64] -; CHECK-NEXT: stp q4, q2, [x1] -; CHECK-NEXT: stp q0, q1, [x1, #32] +; CHECK-NEXT: stp q2, q3, [x1, #96] +; CHECK-NEXT: add z2.s, z6.s, z6.s +; CHECK-NEXT: add z3.s, z4.s, z4.s +; CHECK-NEXT: add z4.s, z7.s, z7.s +; CHECK-NEXT: add z0.s, z0.s, z0.s +; CHECK-NEXT: stp q5, q1, [x1, #64] +; CHECK-NEXT: stp q2, q3, [x1, #32] +; CHECK-NEXT: stp q4, q0, [x1] ; CHECK-NEXT: ret %a = load <32 x i64>, ptr %in %b = trunc <32 x i64> %a to <32 x i32> diff --git a/llvm/test/CodeGen/AArch64/sve-trunc.ll b/llvm/test/CodeGen/AArch64/sve-trunc.ll index 92dfc7396136..0ec6538947c7 100644 --- a/llvm/test/CodeGen/AArch64/sve-trunc.ll +++ b/llvm/test/CodeGen/AArch64/sve-trunc.ll @@ -61,8 +61,8 @@ entry: define @trunc_i64toi1( %in) { ; CHECK-LABEL: trunc_i64toi1: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and z0.d, z0.d, #0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 ; CHECK-NEXT: ret entry: @@ -73,9 +73,9 @@ entry: define @trunc_i64toi1_split( %in) { ; CHECK-LABEL: trunc_i64toi1_split: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and z1.d, z1.d, #0x1 ; CHECK-NEXT: and z0.d, z0.d, #0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: cmpne p1.d, p0/z, z1.d, #0 ; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 ; CHECK-NEXT: uzp1 p0.s, p0.s, p1.s @@ -88,11 +88,11 @@ entry: define @trunc_i64toi1_split2( %in) { ; CHECK-LABEL: trunc_i64toi1_split2: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and z3.d, z3.d, #0x1 ; CHECK-NEXT: and z2.d, z2.d, #0x1 ; CHECK-NEXT: and z1.d, z1.d, #0x1 ; CHECK-NEXT: and z0.d, z0.d, #0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: cmpne p1.d, p0/z, z3.d, #0 ; CHECK-NEXT: cmpne p2.d, p0/z, z2.d, #0 ; CHECK-NEXT: cmpne p3.d, p0/z, z1.d, #0 @@ -111,12 +111,12 @@ define @trunc_i64toi1_split3( %in) { ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 +; CHECK-NEXT: str p7, [sp, #4, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: str p6, [sp, #5, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: str p5, [sp, #6, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and z7.d, z7.d, #0x1 ; CHECK-NEXT: and z6.d, z6.d, #0x1 ; CHECK-NEXT: and z5.d, z5.d, #0x1 @@ -125,23 +125,25 @@ define @trunc_i64toi1_split3( %in) { ; CHECK-NEXT: and z2.d, z2.d, #0x1 ; CHECK-NEXT: and z1.d, z1.d, #0x1 ; CHECK-NEXT: and z0.d, z0.d, #0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: cmpne p1.d, p0/z, z7.d, #0 ; CHECK-NEXT: cmpne p2.d, p0/z, z6.d, #0 ; CHECK-NEXT: cmpne p3.d, p0/z, z5.d, #0 ; CHECK-NEXT: cmpne p4.d, p0/z, z4.d, #0 ; CHECK-NEXT: cmpne p5.d, p0/z, z3.d, #0 ; CHECK-NEXT: cmpne p6.d, p0/z, z2.d, #0 -; CHECK-NEXT: uzp1 p1.s, p2.s, p1.s -; CHECK-NEXT: cmpne p2.d, p0/z, z1.d, #0 +; CHECK-NEXT: cmpne p7.d, p0/z, z1.d, #0 ; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 -; CHECK-NEXT: uzp1 p3.s, p4.s, p3.s -; CHECK-NEXT: uzp1 p4.s, p6.s, p5.s +; CHECK-NEXT: uzp1 p1.s, p2.s, p1.s +; CHECK-NEXT: uzp1 p2.s, p4.s, p3.s +; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload +; CHECK-NEXT: uzp1 p3.s, p6.s, p5.s ; CHECK-NEXT: ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: uzp1 p0.s, p0.s, p2.s +; CHECK-NEXT: uzp1 p0.s, p0.s, p7.s +; CHECK-NEXT: ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload +; CHECK-NEXT: uzp1 p1.h, p2.h, p1.h ; CHECK-NEXT: ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: uzp1 p1.h, p3.h, p1.h -; CHECK-NEXT: uzp1 p0.h, p0.h, p4.h -; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload +; CHECK-NEXT: uzp1 p0.h, p0.h, p3.h ; CHECK-NEXT: uzp1 p0.b, p0.b, p1.b ; CHECK-NEXT: addvl sp, sp, #1 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload @@ -155,8 +157,8 @@ entry: define @trunc_i32toi1( %in) { ; CHECK-LABEL: trunc_i32toi1: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and z0.s, z0.s, #0x1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0 ; CHECK-NEXT: ret entry: @@ -167,8 +169,8 @@ entry: define @trunc_i16toi1( %in) { ; CHECK-LABEL: trunc_i16toi1: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: and z0.h, z0.h, #0x1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #0 ; CHECK-NEXT: ret entry: @@ -179,8 +181,8 @@ entry: define @trunc_i8toi1( %in) { ; CHECK-LABEL: trunc_i8toi1: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: and z0.b, z0.b, #0x1 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: cmpne p0.b, p0/z, z0.b, #0 ; CHECK-NEXT: ret entry: @@ -191,8 +193,8 @@ entry: define @trunc_nxv1i32_to_nxv1i1( %in) { ; CHECK-LABEL: trunc_nxv1i32_to_nxv1i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and z0.s, z0.s, #0x1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0 ; CHECK-NEXT: punpklo p0.h, p0.b ; CHECK-NEXT: punpklo p0.h, p0.b @@ -204,8 +206,8 @@ define @trunc_nxv1i32_to_nxv1i1( %in) { define void @trunc_promoteIntRes( %0, ptr %ptr) { ; CHECK-LABEL: trunc_promoteIntRes: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uzp1 z0.s, z0.s, z1.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: st1h { z0.s }, p0, [x0] ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/AArch64/sve-umulo-sdnode.ll b/llvm/test/CodeGen/AArch64/sve-umulo-sdnode.ll index 36d64725742e..818f37c85ffd 100644 --- a/llvm/test/CodeGen/AArch64/sve-umulo-sdnode.ll +++ b/llvm/test/CodeGen/AArch64/sve-umulo-sdnode.ll @@ -6,9 +6,9 @@ declare { , } @llvm.umul.with.overflow.nxv2i8 define @umulo_nxv2i8( %x, %y) { ; CHECK-LABEL: umulo_nxv2i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and z1.d, z1.d, #0xff ; CHECK-NEXT: and z0.d, z0.d, #0xff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: lsr z1.d, z0.d, #8 ; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0 @@ -26,9 +26,9 @@ declare { , } @llvm.umul.with.overflow.nxv4i8 define @umulo_nxv4i8( %x, %y) { ; CHECK-LABEL: umulo_nxv4i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and z1.s, z1.s, #0xff ; CHECK-NEXT: and z0.s, z0.s, #0xff +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mul z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: lsr z1.s, z0.s, #8 ; CHECK-NEXT: cmpne p0.s, p0/z, z1.s, #0 @@ -46,9 +46,9 @@ declare { , } @llvm.umul.with.overflow.nxv8i8 define @umulo_nxv8i8( %x, %y) { ; CHECK-LABEL: umulo_nxv8i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: and z1.h, z1.h, #0xff ; CHECK-NEXT: and z0.h, z0.h, #0xff +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mul z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: lsr z1.h, z0.h, #8 ; CHECK-NEXT: cmpne p0.h, p0/z, z1.h, #0 @@ -119,9 +119,9 @@ define @umulo_nxv64i8( %x, , } @llvm.umul.with.overflow.nxv2i define @umulo_nxv2i16( %x, %y) { ; CHECK-LABEL: umulo_nxv2i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and z1.d, z1.d, #0xffff ; CHECK-NEXT: and z0.d, z0.d, #0xffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: lsr z1.d, z0.d, #16 ; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0 @@ -163,9 +163,9 @@ declare { , } @llvm.umul.with.overflow.nxv4i define @umulo_nxv4i16( %x, %y) { ; CHECK-LABEL: umulo_nxv4i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and z1.s, z1.s, #0xffff ; CHECK-NEXT: and z0.s, z0.s, #0xffff +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mul z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: lsr z1.s, z0.s, #16 ; CHECK-NEXT: cmpne p0.s, p0/z, z1.s, #0 @@ -236,9 +236,9 @@ define @umulo_nxv32i16( %x, , } @llvm.umul.with.overflow.nxv2i define @umulo_nxv2i32( %x, %y) { ; CHECK-LABEL: umulo_nxv2i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and z1.d, z1.d, #0xffffffff ; CHECK-NEXT: and z0.d, z0.d, #0xffffffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: lsr z1.d, z0.d, #32 ; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0 @@ -333,9 +333,9 @@ define @umulo_nxv16i32( %x, @umulo_nxv8i64( %x, %a, ptr %b) #0 { define void @uzp1_i8_invalid( %a, ptr %b) #0 { ; CHECK-LABEL: uzp1_i8_invalid: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl128 ; CHECK-NEXT: uzp1 z0.b, z0.b, z0.b +; CHECK-NEXT: ptrue p0.b, vl128 ; CHECK-NEXT: st1b { z0.b }, p0, [x0] ; CHECK-NEXT: ret %a.bc = bitcast %a to @@ -141,8 +141,8 @@ define void @uzp1_i16_valid( %a, ptr %b) #0 { define void @uzp1_i16_invalid( %a, ptr %b) #0 { ; CHECK-LABEL: uzp1_i16_invalid: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl64 ; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK-NEXT: ptrue p0.h, vl64 ; CHECK-NEXT: st1h { z0.h }, p0, [x0] ; CHECK-NEXT: ret %a.bc = bitcast %a to @@ -168,8 +168,8 @@ define void @uzp1_i32_valid( %a, ptr %b) #0 { define void @uzp1_i32_invalid( %a, ptr %b) #0 { ; CHECK-LABEL: uzp1_i32_invalid: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl32 ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s +; CHECK-NEXT: ptrue p0.s, vl32 ; CHECK-NEXT: st1w { z0.s }, p0, [x0] ; CHECK-NEXT: ret %a.bc = bitcast %a to @@ -182,8 +182,8 @@ define void @uzp1_i32_invalid( %a, ptr %b) #0 { define void @uzp1_invalid_all( %a, ptr %b) #0 { ; CHECK-LABEL: uzp1_invalid_all: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: st1w { z0.s }, p0, [x0] ; CHECK-NEXT: ret %a.bc = bitcast %a to diff --git a/llvm/test/CodeGen/AArch64/sve-vecreduce-dot.ll b/llvm/test/CodeGen/AArch64/sve-vecreduce-dot.ll index 194d1071301d..91f8f5c2c90d 100644 --- a/llvm/test/CodeGen/AArch64/sve-vecreduce-dot.ll +++ b/llvm/test/CodeGen/AArch64/sve-vecreduce-dot.ll @@ -8,11 +8,11 @@ define i32 @test( %bin.rdx, %bin.rdx2) { ; CHECK-NEXT: sunpklo z5.h, z0.b ; CHECK-NEXT: sunpkhi z0.h, z0.b ; CHECK-NEXT: sunpkhi z2.h, z2.b -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sunpklo z6.h, z1.b ; CHECK-NEXT: sunpkhi z1.h, z1.b ; CHECK-NEXT: sunpklo z7.h, z3.b ; CHECK-NEXT: sunpkhi z3.h, z3.b +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sunpkhi z24.s, z5.h ; CHECK-NEXT: sunpklo z5.s, z5.h ; CHECK-NEXT: sunpklo z25.s, z4.h diff --git a/llvm/test/CodeGen/AArch64/sve-vecreduce-fold.ll b/llvm/test/CodeGen/AArch64/sve-vecreduce-fold.ll index 898090340869..0bdaefdfc2a3 100644 --- a/llvm/test/CodeGen/AArch64/sve-vecreduce-fold.ll +++ b/llvm/test/CodeGen/AArch64/sve-vecreduce-fold.ll @@ -80,8 +80,8 @@ define i1 @reduce_and_insert_subvec_into_var( %in, @test_copysign_v4f32_v4f64( %a, ; CHECK_NO_EXTEND_ROUND-LABEL: test_copysign_v4f32_v4f64: ; CHECK_NO_EXTEND_ROUND: // %bb.0: ; CHECK_NO_EXTEND_ROUND-NEXT: ptrue p0.d +; CHECK_NO_EXTEND_ROUND-NEXT: mov z3.s, #0x7fffffff ; CHECK_NO_EXTEND_ROUND-NEXT: fcvt z2.s, p0/m, z2.d ; CHECK_NO_EXTEND_ROUND-NEXT: fcvt z1.s, p0/m, z1.d ; CHECK_NO_EXTEND_ROUND-NEXT: uzp1 z1.s, z1.s, z2.s -; CHECK_NO_EXTEND_ROUND-NEXT: mov z2.s, #0x7fffffff -; CHECK_NO_EXTEND_ROUND-NEXT: bsl z0.d, z0.d, z1.d, z2.d +; CHECK_NO_EXTEND_ROUND-NEXT: bsl z0.d, z0.d, z1.d, z3.d ; CHECK_NO_EXTEND_ROUND-NEXT: ret ; ; CHECK_EXTEND_ROUND-LABEL: test_copysign_v4f32_v4f64: @@ -107,9 +107,9 @@ declare @llvm.copysign.v2f64( %a, @test_copysign_v4f64_v4f32( %a, %b) #0 { ; CHECK_NO_EXTEND_ROUND-LABEL: test_copysign_v4f64_v4f32: ; CHECK_NO_EXTEND_ROUND: // %bb.0: -; CHECK_NO_EXTEND_ROUND-NEXT: ptrue p0.d ; CHECK_NO_EXTEND_ROUND-NEXT: uunpkhi z3.d, z2.s ; CHECK_NO_EXTEND_ROUND-NEXT: uunpklo z2.d, z2.s +; CHECK_NO_EXTEND_ROUND-NEXT: ptrue p0.d ; CHECK_NO_EXTEND_ROUND-NEXT: mov z4.d, #0x7fffffffffffffff ; CHECK_NO_EXTEND_ROUND-NEXT: fcvt z3.d, p0/m, z3.s ; CHECK_NO_EXTEND_ROUND-NEXT: fcvt z2.d, p0/m, z2.s @@ -119,9 +119,9 @@ define @test_copysign_v4f64_v4f32( %a ; ; CHECK_EXTEND_ROUND-LABEL: test_copysign_v4f64_v4f32: ; CHECK_EXTEND_ROUND: // %bb.0: -; CHECK_EXTEND_ROUND-NEXT: ptrue p0.d ; CHECK_EXTEND_ROUND-NEXT: uunpkhi z3.d, z2.s ; CHECK_EXTEND_ROUND-NEXT: uunpklo z2.d, z2.s +; CHECK_EXTEND_ROUND-NEXT: ptrue p0.d ; CHECK_EXTEND_ROUND-NEXT: mov z4.d, #0x7fffffffffffffff ; CHECK_EXTEND_ROUND-NEXT: fcvt z2.d, p0/m, z2.s ; CHECK_EXTEND_ROUND-NEXT: fcvt z3.d, p0/m, z3.s @@ -176,11 +176,11 @@ define @test_copysign_v4f16_v4f64( %a, @test_copysign_v8f16_v8f32( %a, %arg1){ ; CHECK-LABEL: wide_add_shift_add_rshrnb_b: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: rshrnb z1.b, z1.h, #6 ; CHECK-NEXT: rshrnb z0.b, z0.h, #6 +; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: ld1b { z2.b }, p0/z, [x0, x1] ; CHECK-NEXT: uzp1 z0.b, z0.b, z1.b -; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0, x1] -; CHECK-NEXT: add z0.b, z1.b, z0.b +; CHECK-NEXT: add z0.b, z2.b, z0.b ; CHECK-NEXT: st1b { z0.b }, p0, [x0, x1] ; CHECK-NEXT: ret %1 = add %arg1, splat (i16 32) @@ -141,12 +141,12 @@ define void @wide_add_shift_add_rshrnb_b(ptr %dest, i64 %index, %arg1){ ; CHECK-LABEL: wide_add_shift_add_rshrnb_h: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: rshrnb z1.h, z1.s, #6 ; CHECK-NEXT: rshrnb z0.h, z0.s, #6 +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: ld1h { z2.h }, p0/z, [x0, x1, lsl #1] ; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h -; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0, x1, lsl #1] -; CHECK-NEXT: add z0.h, z1.h, z0.h +; CHECK-NEXT: add z0.h, z2.h, z0.h ; CHECK-NEXT: st1h { z0.h }, p0, [x0, x1, lsl #1] ; CHECK-NEXT: ret %1 = add %arg1, splat (i32 32) @@ -162,12 +162,12 @@ define void @wide_add_shift_add_rshrnb_h(ptr %dest, i64 %index, %arg1){ ; CHECK-LABEL: wide_add_shift_add_rshrnb_d: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: rshrnb z1.s, z1.d, #32 ; CHECK-NEXT: rshrnb z0.s, z0.d, #32 +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: ld1w { z2.s }, p0/z, [x0, x1, lsl #2] ; CHECK-NEXT: uzp1 z0.s, z0.s, z1.s -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0, x1, lsl #2] -; CHECK-NEXT: add z0.s, z1.s, z0.s +; CHECK-NEXT: add z0.s, z2.s, z0.s ; CHECK-NEXT: st1w { z0.s }, p0, [x0, x1, lsl #2] ; CHECK-NEXT: ret %1 = add %arg1, splat (i64 2147483648) @@ -188,11 +188,11 @@ define void @neg_wide_add_shift_add_rshrnb_d(ptr %dest, i64 %index, %arg1, splat (i64 140737488355328) diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-int-arith-imm.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-int-arith-imm.ll index 8bfcd088e1a8..500973d053f5 100644 --- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-int-arith-imm.ll +++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-int-arith-imm.ll @@ -437,8 +437,8 @@ define @uqsub_i32_ptrue_all_h( %a) #0 { define @uqsub_i32_ptrue_all_d( %a) #0 { ; CHECK-LABEL: uqsub_i32_ptrue_all_d: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.s, #1 // =0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: uqsub z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %pg.d = tail call @llvm.aarch64.sve.ptrue.nxv2i1(i32 31) diff --git a/llvm/test/CodeGen/AArch64/sve2-rsh.ll b/llvm/test/CodeGen/AArch64/sve2-rsh.ll index 516ef3bd581e..9addd16f8929 100644 --- a/llvm/test/CodeGen/AArch64/sve2-rsh.ll +++ b/llvm/test/CodeGen/AArch64/sve2-rsh.ll @@ -18,8 +18,8 @@ define @neg_urshr_1( %x) { define @neg_urshr_2( %x, %y) { ; CHECK-LABEL: neg_urshr_2: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: add z0.d, z0.d, #32 // =0x20 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: lsr z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret %add = add nuw nsw %x, splat (i64 32) diff --git a/llvm/test/CodeGen/AArch64/sve2-xar.ll b/llvm/test/CodeGen/AArch64/sve2-xar.ll index e297ade6b9ae..e5a240b7a53f 100644 --- a/llvm/test/CodeGen/AArch64/sve2-xar.ll +++ b/llvm/test/CodeGen/AArch64/sve2-xar.ll @@ -152,9 +152,9 @@ define @xar_nxv2i64_l_neg1( %x, , } @sel_x2_i8(target("aarch64.svc ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z5.d, z4.d ; CHECK-NEXT: mov z7.d, z2.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z4.d, z3.d ; CHECK-NEXT: mov z6.d, z1.d +; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: sel { z0.b, z1.b }, pn8, { z6.b, z7.b }, { z4.b, z5.b } ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: addvl sp, sp, #1 @@ -28,12 +28,12 @@ define { , } @sel_x2_i16(target("aarch64.sv ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z5.d, z4.d ; CHECK-NEXT: mov z7.d, z2.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z4.d, z3.d ; CHECK-NEXT: mov z6.d, z1.d +; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: sel { z0.h, z1.h }, pn8, { z6.h, z7.h }, { z4.h, z5.h } ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: addvl sp, sp, #1 @@ -48,12 +48,12 @@ define { , } @sel_x2_f16(target("aarch64. ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z5.d, z4.d ; CHECK-NEXT: mov z7.d, z2.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z4.d, z3.d ; CHECK-NEXT: mov z6.d, z1.d +; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: sel { z0.h, z1.h }, pn8, { z6.h, z7.h }, { z4.h, z5.h } ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: addvl sp, sp, #1 @@ -68,12 +68,12 @@ define { , } @sel_x2_bf16(target("aar ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z5.d, z4.d ; CHECK-NEXT: mov z7.d, z2.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z4.d, z3.d ; CHECK-NEXT: mov z6.d, z1.d +; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: sel { z0.h, z1.h }, pn8, { z6.h, z7.h }, { z4.h, z5.h } ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: addvl sp, sp, #1 @@ -88,12 +88,12 @@ define { , } @sel_x2_i32(target("aarch64.sv ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z5.d, z4.d ; CHECK-NEXT: mov z7.d, z2.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z4.d, z3.d ; CHECK-NEXT: mov z6.d, z1.d +; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: sel { z0.s, z1.s }, pn8, { z6.s, z7.s }, { z4.s, z5.s } ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: addvl sp, sp, #1 @@ -108,12 +108,12 @@ define { , } @sel_x2_f32(target("aarch6 ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z5.d, z4.d ; CHECK-NEXT: mov z7.d, z2.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z4.d, z3.d ; CHECK-NEXT: mov z6.d, z1.d +; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: sel { z0.s, z1.s }, pn8, { z6.s, z7.s }, { z4.s, z5.s } ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: addvl sp, sp, #1 @@ -128,12 +128,12 @@ define { , } @sel_x2_i64(target("aarch64.sv ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z5.d, z4.d ; CHECK-NEXT: mov z7.d, z2.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z4.d, z3.d ; CHECK-NEXT: mov z6.d, z1.d +; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: sel { z0.d, z1.d }, pn8, { z6.d, z7.d }, { z4.d, z5.d } ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: addvl sp, sp, #1 @@ -148,12 +148,12 @@ define { , } @sel_x2_f64(target("aarc ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z5.d, z4.d ; CHECK-NEXT: mov z7.d, z2.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z4.d, z3.d ; CHECK-NEXT: mov z6.d, z1.d +; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: sel { z0.d, z1.d }, pn8, { z6.d, z7.d }, { z4.d, z5.d } ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: addvl sp, sp, #1 diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-selx4.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-selx4.ll index df504362680b..3a21eaead5f7 100644 --- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-selx4.ll +++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-selx4.ll @@ -8,17 +8,17 @@ define { , , , , , , , , , , , , , , , , , , , , , , , , %unused, %zn0, %unused, %zn0, %unused, %zn0, %unused, %zn0, %unused, %zn0, %unused, %zn0, ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 +; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b -; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: st1h { z2.h, z3.h }, pn8, [x0] ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload @@ -117,9 +117,9 @@ define void @st1_x2_f32( %unused, %zn0, < ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 +; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b -; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: st1w { z2.s, z3.s }, pn8, [x0] ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload @@ -135,9 +135,9 @@ define void @st1_x2_f64( %unused, %zn0, ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 +; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b -; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: st1d { z2.d, z3.d }, pn8, [x0] ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload @@ -153,8 +153,8 @@ define void @st1_x4_i8( %unused, %zn0, %unused, %zn0, %unused, %zn0, %unused, %zn0, %unused, %zn0, %unused, %zn0, ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z7.d, z4.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z6.d, z3.d ; CHECK-NEXT: mov z5.d, z2.d @@ -273,8 +273,8 @@ define void @st1_x4_f32( %unused, %zn0, < ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z7.d, z4.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z6.d, z3.d ; CHECK-NEXT: mov z5.d, z2.d @@ -293,8 +293,8 @@ define void @st1_x4_f64( %unused, %zn0, ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z7.d, z4.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z6.d, z3.d ; CHECK-NEXT: mov z5.d, z2.d @@ -315,9 +315,9 @@ define void @stnt1_x2_i8( %unused, %zn0, %unused, %zn0, < ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 +; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b -; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: stnt1h { z2.h, z3.h }, pn8, [x0] ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload @@ -351,9 +351,9 @@ define void @stnt1_x2_i32( %unused, %zn0, < ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 +; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b -; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: stnt1w { z2.s, z3.s }, pn8, [x0] ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload @@ -369,9 +369,9 @@ define void @stnt1_x2_i64( %unused, %zn0, < ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 +; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b -; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: stnt1d { z2.d, z3.d }, pn8, [x0] ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload @@ -387,9 +387,9 @@ define void @stnt1_x2_f16( %unused, %zn0, ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 +; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b -; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: stnt1h { z2.h, z3.h }, pn8, [x0] ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload @@ -405,9 +405,9 @@ define void @stnt1_x2_bf16( %unused, %zn ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 +; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b -; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: stnt1h { z2.h, z3.h }, pn8, [x0] ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload @@ -423,9 +423,9 @@ define void @stnt1_x2_f32( %unused, %zn0, ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 +; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b -; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: stnt1w { z2.s, z3.s }, pn8, [x0] ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload @@ -441,9 +441,9 @@ define void @stnt1_x2_f64( %unused, %zn0 ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 +; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b -; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: stnt1d { z2.d, z3.d }, pn8, [x0] ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload @@ -459,8 +459,8 @@ define void @stnt1_x4_i8( %unused, %zn0, %unused, %zn0, < ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z7.d, z4.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z6.d, z3.d ; CHECK-NEXT: mov z5.d, z2.d @@ -499,8 +499,8 @@ define void @stnt1_x4_i32( %unused, %zn0, < ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z7.d, z4.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z6.d, z3.d ; CHECK-NEXT: mov z5.d, z2.d @@ -519,8 +519,8 @@ define void @stnt1_x4_i64( %unused, %zn0, < ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z7.d, z4.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z6.d, z3.d ; CHECK-NEXT: mov z5.d, z2.d @@ -539,8 +539,8 @@ define void @stnt1_x4_f16( %unused, %zn0, ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z7.d, z4.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z6.d, z3.d ; CHECK-NEXT: mov z5.d, z2.d @@ -559,8 +559,8 @@ define void @stnt1_x4_bf16( %unused, %zn ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z7.d, z4.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z6.d, z3.d ; CHECK-NEXT: mov z5.d, z2.d @@ -579,8 +579,8 @@ define void @stnt1_x4_f32( %unused, %zn0, ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z7.d, z4.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z6.d, z3.d ; CHECK-NEXT: mov z5.d, z2.d @@ -599,8 +599,8 @@ define void @stnt1_x4_f64( %unused, %zn0 ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z7.d, z4.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z6.d, z3.d ; CHECK-NEXT: mov z5.d, z2.d diff --git a/llvm/test/CodeGen/AArch64/uadd_sat_vec.ll b/llvm/test/CodeGen/AArch64/uadd_sat_vec.ll index 30ff70088454..16521834090b 100644 --- a/llvm/test/CodeGen/AArch64/uadd_sat_vec.ll +++ b/llvm/test/CodeGen/AArch64/uadd_sat_vec.ll @@ -147,11 +147,11 @@ define void @v8i8(ptr %px, ptr %py, ptr %pz) nounwind { define void @v4i8(ptr %px, ptr %py, ptr %pz) nounwind { ; CHECK-SD-LABEL: v4i8: ; CHECK-SD: // %bb.0: -; CHECK-SD-NEXT: ldr s1, [x0] -; CHECK-SD-NEXT: ldr s2, [x1] -; CHECK-SD-NEXT: movi d0, #0xff00ff00ff00ff -; CHECK-SD-NEXT: uaddl v1.8h, v1.8b, v2.8b -; CHECK-SD-NEXT: umin v0.4h, v1.4h, v0.4h +; CHECK-SD-NEXT: ldr s0, [x0] +; CHECK-SD-NEXT: ldr s1, [x1] +; CHECK-SD-NEXT: movi d2, #0xff00ff00ff00ff +; CHECK-SD-NEXT: uaddl v0.8h, v0.8b, v1.8b +; CHECK-SD-NEXT: umin v0.4h, v0.4h, v2.4h ; CHECK-SD-NEXT: uzp1 v0.8b, v0.8b, v0.8b ; CHECK-SD-NEXT: str s0, [x2] ; CHECK-SD-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/urem-seteq-vec-nonzero.ll b/llvm/test/CodeGen/AArch64/urem-seteq-vec-nonzero.ll index b31ce94cdaae..d5f1febaeb7d 100644 --- a/llvm/test/CodeGen/AArch64/urem-seteq-vec-nonzero.ll +++ b/llvm/test/CodeGen/AArch64/urem-seteq-vec-nonzero.ll @@ -52,11 +52,11 @@ define <4 x i1> @t32_6_part0(<4 x i32> %X) nounwind { ; CHECK-NEXT: sub v0.4s, v0.4s, v1.4s ; CHECK-NEXT: dup v1.4s, w8 ; CHECK-NEXT: mul v0.4s, v0.4s, v1.4s -; CHECK-NEXT: shl v1.4s, v0.4s, #31 -; CHECK-NEXT: usra v1.4s, v0.4s, #1 -; CHECK-NEXT: movi v0.16b, #170 -; CHECK-NEXT: fneg v0.4s, v0.4s -; CHECK-NEXT: cmhs v0.4s, v0.4s, v1.4s +; CHECK-NEXT: movi v1.16b, #170 +; CHECK-NEXT: shl v2.4s, v0.4s, #31 +; CHECK-NEXT: fneg v1.4s, v1.4s +; CHECK-NEXT: usra v2.4s, v0.4s, #1 +; CHECK-NEXT: cmhs v0.4s, v1.4s, v2.4s ; CHECK-NEXT: xtn v0.4h, v0.4s ; CHECK-NEXT: ret %urem = urem <4 x i32> %X, diff --git a/llvm/test/CodeGen/AArch64/vec_uaddo.ll b/llvm/test/CodeGen/AArch64/vec_uaddo.ll index 00609b0df9b4..37c6374215d8 100644 --- a/llvm/test/CodeGen/AArch64/vec_uaddo.ll +++ b/llvm/test/CodeGen/AArch64/vec_uaddo.ll @@ -100,9 +100,9 @@ define <6 x i32> @uaddo_v6i32(<6 x i32> %a0, <6 x i32> %a1, ptr %p2) nounwind { ; CHECK-NEXT: mov v0.s[3], w3 ; CHECK-NEXT: cmhi v3.4s, v3.4s, v2.4s ; CHECK-NEXT: str d2, [x8, #16] -; CHECK-NEXT: add v1.4s, v0.4s, v1.4s ; CHECK-NEXT: mov w5, v3.s[1] ; CHECK-NEXT: fmov w4, s3 +; CHECK-NEXT: add v1.4s, v0.4s, v1.4s ; CHECK-NEXT: cmhi v0.4s, v0.4s, v1.4s ; CHECK-NEXT: str q1, [x8] ; CHECK-NEXT: mov w1, v0.s[1] @@ -248,10 +248,10 @@ define <4 x i32> @uaddo_v4i1(<4 x i1> %a0, <4 x i1> %a1, ptr %p2) nounwind { ; CHECK-NEXT: eor v2.8b, v0.8b, v1.8b ; CHECK-NEXT: and v0.8b, v0.8b, v1.8b ; CHECK-NEXT: adrp x8, .LCPI10_0 -; CHECK-NEXT: shl v2.4h, v2.4h, #15 -; CHECK-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-NEXT: cmlt v1.4h, v2.4h, #0 +; CHECK-NEXT: shl v1.4h, v2.4h, #15 ; CHECK-NEXT: ldr d2, [x8, :lo12:.LCPI10_0] +; CHECK-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-NEXT: cmlt v1.4h, v1.4h, #0 ; CHECK-NEXT: shl v0.4s, v0.4s, #31 ; CHECK-NEXT: and v1.8b, v1.8b, v2.8b ; CHECK-NEXT: cmlt v0.4s, v0.4s, #0 diff --git a/llvm/test/CodeGen/AArch64/vecreduce-add.ll b/llvm/test/CodeGen/AArch64/vecreduce-add.ll index 66ef436f48c6..3254c5ebe9c6 100644 --- a/llvm/test/CodeGen/AArch64/vecreduce-add.ll +++ b/llvm/test/CodeGen/AArch64/vecreduce-add.ll @@ -2177,65 +2177,65 @@ define i32 @test_udot_v48i8(ptr %p1, ptr %p2) { ; CHECK-GI-BASE-LABEL: test_udot_v48i8: ; CHECK-GI-BASE: // %bb.0: // %entry ; CHECK-GI-BASE-NEXT: ldp q0, q3, [x1] -; CHECK-GI-BASE-NEXT: ldr q6, [x1, #32] +; CHECK-GI-BASE-NEXT: ldr q6, [x0, #32] ; CHECK-GI-BASE-NEXT: ldp q1, q2, [x0] -; CHECK-GI-BASE-NEXT: ldr q17, [x0, #32] +; CHECK-GI-BASE-NEXT: ldr q7, [x1, #32] +; CHECK-GI-BASE-NEXT: ushll v20.8h, v6.8b, #0 +; CHECK-GI-BASE-NEXT: ushll2 v6.8h, v6.16b, #0 ; CHECK-GI-BASE-NEXT: ushll v4.8h, v0.8b, #0 ; CHECK-GI-BASE-NEXT: ushll2 v0.8h, v0.16b, #0 -; CHECK-GI-BASE-NEXT: ushll v7.8h, v3.8b, #0 +; CHECK-GI-BASE-NEXT: ushll v16.8h, v3.8b, #0 ; CHECK-GI-BASE-NEXT: ushll v5.8h, v1.8b, #0 ; CHECK-GI-BASE-NEXT: ushll2 v1.8h, v1.16b, #0 -; CHECK-GI-BASE-NEXT: ushll v16.8h, v2.8b, #0 +; CHECK-GI-BASE-NEXT: ushll v17.8h, v2.8b, #0 ; CHECK-GI-BASE-NEXT: ushll2 v3.8h, v3.16b, #0 ; CHECK-GI-BASE-NEXT: ushll2 v2.8h, v2.16b, #0 ; CHECK-GI-BASE-NEXT: umull v18.4s, v4.4h, v5.4h ; CHECK-GI-BASE-NEXT: umull2 v4.4s, v4.8h, v5.8h -; CHECK-GI-BASE-NEXT: umull2 v19.4s, v0.8h, v1.8h -; CHECK-GI-BASE-NEXT: umull v20.4s, v7.4h, v16.4h -; CHECK-GI-BASE-NEXT: umull v0.4s, v0.4h, v1.4h -; CHECK-GI-BASE-NEXT: ushll v5.8h, v6.8b, #0 -; CHECK-GI-BASE-NEXT: ushll v1.8h, v17.8b, #0 -; CHECK-GI-BASE-NEXT: umull2 v7.4s, v7.8h, v16.8h -; CHECK-GI-BASE-NEXT: ushll2 v6.8h, v6.16b, #0 -; CHECK-GI-BASE-NEXT: ushll2 v17.8h, v17.16b, #0 -; CHECK-GI-BASE-NEXT: addv s16, v18.4s -; CHECK-GI-BASE-NEXT: addv s4, v4.4s -; CHECK-GI-BASE-NEXT: umull v18.4s, v3.4h, v2.4h +; CHECK-GI-BASE-NEXT: umull v5.4s, v0.4h, v1.4h +; CHECK-GI-BASE-NEXT: umull2 v0.4s, v0.8h, v1.8h +; CHECK-GI-BASE-NEXT: umull v19.4s, v16.4h, v17.4h +; CHECK-GI-BASE-NEXT: ushll v1.8h, v7.8b, #0 +; CHECK-GI-BASE-NEXT: umull2 v16.4s, v16.8h, v17.8h +; CHECK-GI-BASE-NEXT: umull v17.4s, v3.4h, v2.4h ; CHECK-GI-BASE-NEXT: umull2 v2.4s, v3.8h, v2.8h -; CHECK-GI-BASE-NEXT: addv s3, v19.4s -; CHECK-GI-BASE-NEXT: umull v19.4s, v5.4h, v1.4h -; CHECK-GI-BASE-NEXT: umull2 v1.4s, v5.8h, v1.8h -; CHECK-GI-BASE-NEXT: addv s5, v20.4s +; CHECK-GI-BASE-NEXT: ushll2 v7.8h, v7.16b, #0 +; CHECK-GI-BASE-NEXT: addv s18, v18.4s +; CHECK-GI-BASE-NEXT: addv s4, v4.4s +; CHECK-GI-BASE-NEXT: addv s5, v5.4s ; CHECK-GI-BASE-NEXT: addv s0, v0.4s -; CHECK-GI-BASE-NEXT: addv s7, v7.4s -; CHECK-GI-BASE-NEXT: umull v20.4s, v6.4h, v17.4h -; CHECK-GI-BASE-NEXT: umull2 v6.4s, v6.8h, v17.8h -; CHECK-GI-BASE-NEXT: fmov w8, s16 -; CHECK-GI-BASE-NEXT: fmov w9, s4 -; CHECK-GI-BASE-NEXT: fmov w10, s3 -; CHECK-GI-BASE-NEXT: addv s3, v18.4s +; CHECK-GI-BASE-NEXT: addv s19, v19.4s +; CHECK-GI-BASE-NEXT: umull v3.4s, v1.4h, v20.4h ; CHECK-GI-BASE-NEXT: addv s2, v2.4s -; CHECK-GI-BASE-NEXT: fmov w11, s5 -; CHECK-GI-BASE-NEXT: addv s4, v19.4s +; CHECK-GI-BASE-NEXT: umull2 v1.4s, v1.8h, v20.8h +; CHECK-GI-BASE-NEXT: umull v20.4s, v7.4h, v6.4h +; CHECK-GI-BASE-NEXT: fmov w8, s18 +; CHECK-GI-BASE-NEXT: fmov w9, s4 +; CHECK-GI-BASE-NEXT: fmov w10, s5 +; CHECK-GI-BASE-NEXT: fmov w11, s0 +; CHECK-GI-BASE-NEXT: fmov w12, s19 +; CHECK-GI-BASE-NEXT: addv s4, v16.4s +; CHECK-GI-BASE-NEXT: addv s5, v17.4s +; CHECK-GI-BASE-NEXT: addv s3, v3.4s +; CHECK-GI-BASE-NEXT: umull2 v0.4s, v7.8h, v6.8h ; CHECK-GI-BASE-NEXT: add w8, w8, w9 -; CHECK-GI-BASE-NEXT: fmov w9, s0 -; CHECK-GI-BASE-NEXT: addv s0, v1.4s -; CHECK-GI-BASE-NEXT: addv s1, v20.4s -; CHECK-GI-BASE-NEXT: addv s5, v6.4s -; CHECK-GI-BASE-NEXT: add w10, w10, w11 -; CHECK-GI-BASE-NEXT: fmov w11, s3 +; CHECK-GI-BASE-NEXT: addv s1, v1.4s +; CHECK-GI-BASE-NEXT: add w9, w11, w12 +; CHECK-GI-BASE-NEXT: add w8, w8, w10 +; CHECK-GI-BASE-NEXT: fmov w10, s4 +; CHECK-GI-BASE-NEXT: fmov w11, s5 ; CHECK-GI-BASE-NEXT: fmov w12, s2 -; CHECK-GI-BASE-NEXT: add w8, w8, w9 -; CHECK-GI-BASE-NEXT: fmov w9, s7 -; CHECK-GI-BASE-NEXT: add w9, w10, w9 +; CHECK-GI-BASE-NEXT: addv s4, v20.4s +; CHECK-GI-BASE-NEXT: addv s0, v0.4s +; CHECK-GI-BASE-NEXT: add w9, w9, w10 ; CHECK-GI-BASE-NEXT: add w10, w11, w12 -; CHECK-GI-BASE-NEXT: fmov w11, s4 +; CHECK-GI-BASE-NEXT: fmov w11, s3 ; CHECK-GI-BASE-NEXT: add w8, w8, w9 ; CHECK-GI-BASE-NEXT: add w9, w10, w11 -; CHECK-GI-BASE-NEXT: fmov w10, s0 -; CHECK-GI-BASE-NEXT: fmov w11, s5 -; CHECK-GI-BASE-NEXT: add w9, w9, w10 ; CHECK-GI-BASE-NEXT: fmov w10, s1 +; CHECK-GI-BASE-NEXT: fmov w11, s0 +; CHECK-GI-BASE-NEXT: add w9, w9, w10 +; CHECK-GI-BASE-NEXT: fmov w10, s4 ; CHECK-GI-BASE-NEXT: add w8, w8, w9 ; CHECK-GI-BASE-NEXT: add w9, w10, w11 ; CHECK-GI-BASE-NEXT: add w0, w8, w9 @@ -2527,65 +2527,65 @@ define i32 @test_sdot_v48i8(ptr %p1, ptr %p2) { ; CHECK-GI-BASE-LABEL: test_sdot_v48i8: ; CHECK-GI-BASE: // %bb.0: // %entry ; CHECK-GI-BASE-NEXT: ldp q0, q3, [x1] -; CHECK-GI-BASE-NEXT: ldr q6, [x1, #32] +; CHECK-GI-BASE-NEXT: ldr q6, [x0, #32] ; CHECK-GI-BASE-NEXT: ldp q1, q2, [x0] -; CHECK-GI-BASE-NEXT: ldr q17, [x0, #32] +; CHECK-GI-BASE-NEXT: ldr q7, [x1, #32] +; CHECK-GI-BASE-NEXT: sshll v20.8h, v6.8b, #0 +; CHECK-GI-BASE-NEXT: sshll2 v6.8h, v6.16b, #0 ; CHECK-GI-BASE-NEXT: sshll v4.8h, v0.8b, #0 ; CHECK-GI-BASE-NEXT: sshll2 v0.8h, v0.16b, #0 -; CHECK-GI-BASE-NEXT: sshll v7.8h, v3.8b, #0 +; CHECK-GI-BASE-NEXT: sshll v16.8h, v3.8b, #0 ; CHECK-GI-BASE-NEXT: sshll v5.8h, v1.8b, #0 ; CHECK-GI-BASE-NEXT: sshll2 v1.8h, v1.16b, #0 -; CHECK-GI-BASE-NEXT: sshll v16.8h, v2.8b, #0 +; CHECK-GI-BASE-NEXT: sshll v17.8h, v2.8b, #0 ; CHECK-GI-BASE-NEXT: sshll2 v3.8h, v3.16b, #0 ; CHECK-GI-BASE-NEXT: sshll2 v2.8h, v2.16b, #0 ; CHECK-GI-BASE-NEXT: smull v18.4s, v4.4h, v5.4h ; CHECK-GI-BASE-NEXT: smull2 v4.4s, v4.8h, v5.8h -; CHECK-GI-BASE-NEXT: smull2 v19.4s, v0.8h, v1.8h -; CHECK-GI-BASE-NEXT: smull v20.4s, v7.4h, v16.4h -; CHECK-GI-BASE-NEXT: smull v0.4s, v0.4h, v1.4h -; CHECK-GI-BASE-NEXT: sshll v5.8h, v6.8b, #0 -; CHECK-GI-BASE-NEXT: sshll v1.8h, v17.8b, #0 -; CHECK-GI-BASE-NEXT: smull2 v7.4s, v7.8h, v16.8h -; CHECK-GI-BASE-NEXT: sshll2 v6.8h, v6.16b, #0 -; CHECK-GI-BASE-NEXT: sshll2 v17.8h, v17.16b, #0 -; CHECK-GI-BASE-NEXT: addv s16, v18.4s -; CHECK-GI-BASE-NEXT: addv s4, v4.4s -; CHECK-GI-BASE-NEXT: smull v18.4s, v3.4h, v2.4h +; CHECK-GI-BASE-NEXT: smull v5.4s, v0.4h, v1.4h +; CHECK-GI-BASE-NEXT: smull2 v0.4s, v0.8h, v1.8h +; CHECK-GI-BASE-NEXT: smull v19.4s, v16.4h, v17.4h +; CHECK-GI-BASE-NEXT: sshll v1.8h, v7.8b, #0 +; CHECK-GI-BASE-NEXT: smull2 v16.4s, v16.8h, v17.8h +; CHECK-GI-BASE-NEXT: smull v17.4s, v3.4h, v2.4h ; CHECK-GI-BASE-NEXT: smull2 v2.4s, v3.8h, v2.8h -; CHECK-GI-BASE-NEXT: addv s3, v19.4s -; CHECK-GI-BASE-NEXT: smull v19.4s, v5.4h, v1.4h -; CHECK-GI-BASE-NEXT: smull2 v1.4s, v5.8h, v1.8h -; CHECK-GI-BASE-NEXT: addv s5, v20.4s +; CHECK-GI-BASE-NEXT: sshll2 v7.8h, v7.16b, #0 +; CHECK-GI-BASE-NEXT: addv s18, v18.4s +; CHECK-GI-BASE-NEXT: addv s4, v4.4s +; CHECK-GI-BASE-NEXT: addv s5, v5.4s ; CHECK-GI-BASE-NEXT: addv s0, v0.4s -; CHECK-GI-BASE-NEXT: addv s7, v7.4s -; CHECK-GI-BASE-NEXT: smull v20.4s, v6.4h, v17.4h -; CHECK-GI-BASE-NEXT: smull2 v6.4s, v6.8h, v17.8h -; CHECK-GI-BASE-NEXT: fmov w8, s16 -; CHECK-GI-BASE-NEXT: fmov w9, s4 -; CHECK-GI-BASE-NEXT: fmov w10, s3 -; CHECK-GI-BASE-NEXT: addv s3, v18.4s +; CHECK-GI-BASE-NEXT: addv s19, v19.4s +; CHECK-GI-BASE-NEXT: smull v3.4s, v1.4h, v20.4h ; CHECK-GI-BASE-NEXT: addv s2, v2.4s -; CHECK-GI-BASE-NEXT: fmov w11, s5 -; CHECK-GI-BASE-NEXT: addv s4, v19.4s +; CHECK-GI-BASE-NEXT: smull2 v1.4s, v1.8h, v20.8h +; CHECK-GI-BASE-NEXT: smull v20.4s, v7.4h, v6.4h +; CHECK-GI-BASE-NEXT: fmov w8, s18 +; CHECK-GI-BASE-NEXT: fmov w9, s4 +; CHECK-GI-BASE-NEXT: fmov w10, s5 +; CHECK-GI-BASE-NEXT: fmov w11, s0 +; CHECK-GI-BASE-NEXT: fmov w12, s19 +; CHECK-GI-BASE-NEXT: addv s4, v16.4s +; CHECK-GI-BASE-NEXT: addv s5, v17.4s +; CHECK-GI-BASE-NEXT: addv s3, v3.4s +; CHECK-GI-BASE-NEXT: smull2 v0.4s, v7.8h, v6.8h ; CHECK-GI-BASE-NEXT: add w8, w8, w9 -; CHECK-GI-BASE-NEXT: fmov w9, s0 -; CHECK-GI-BASE-NEXT: addv s0, v1.4s -; CHECK-GI-BASE-NEXT: addv s1, v20.4s -; CHECK-GI-BASE-NEXT: addv s5, v6.4s -; CHECK-GI-BASE-NEXT: add w10, w10, w11 -; CHECK-GI-BASE-NEXT: fmov w11, s3 +; CHECK-GI-BASE-NEXT: addv s1, v1.4s +; CHECK-GI-BASE-NEXT: add w9, w11, w12 +; CHECK-GI-BASE-NEXT: add w8, w8, w10 +; CHECK-GI-BASE-NEXT: fmov w10, s4 +; CHECK-GI-BASE-NEXT: fmov w11, s5 ; CHECK-GI-BASE-NEXT: fmov w12, s2 -; CHECK-GI-BASE-NEXT: add w8, w8, w9 -; CHECK-GI-BASE-NEXT: fmov w9, s7 -; CHECK-GI-BASE-NEXT: add w9, w10, w9 +; CHECK-GI-BASE-NEXT: addv s4, v20.4s +; CHECK-GI-BASE-NEXT: addv s0, v0.4s +; CHECK-GI-BASE-NEXT: add w9, w9, w10 ; CHECK-GI-BASE-NEXT: add w10, w11, w12 -; CHECK-GI-BASE-NEXT: fmov w11, s4 +; CHECK-GI-BASE-NEXT: fmov w11, s3 ; CHECK-GI-BASE-NEXT: add w8, w8, w9 ; CHECK-GI-BASE-NEXT: add w9, w10, w11 -; CHECK-GI-BASE-NEXT: fmov w10, s0 -; CHECK-GI-BASE-NEXT: fmov w11, s5 -; CHECK-GI-BASE-NEXT: add w9, w9, w10 ; CHECK-GI-BASE-NEXT: fmov w10, s1 +; CHECK-GI-BASE-NEXT: fmov w11, s0 +; CHECK-GI-BASE-NEXT: add w9, w9, w10 +; CHECK-GI-BASE-NEXT: fmov w10, s4 ; CHECK-GI-BASE-NEXT: add w8, w8, w9 ; CHECK-GI-BASE-NEXT: add w9, w10, w11 ; CHECK-GI-BASE-NEXT: add w0, w8, w9 @@ -2640,13 +2640,13 @@ define i32 @test_udot_v8i8_multi_use(<8 x i8> %a, <8 x i8> %b) { ; CHECK-SD-DOT-LABEL: test_udot_v8i8_multi_use: ; CHECK-SD-DOT: // %bb.0: // %entry ; CHECK-SD-DOT-NEXT: movi v2.2d, #0000000000000000 +; CHECK-SD-DOT-NEXT: ushll v3.8h, v0.8b, #0 +; CHECK-SD-DOT-NEXT: ushll v4.8h, v1.8b, #0 ; CHECK-SD-DOT-NEXT: udot v2.2s, v1.8b, v0.8b -; CHECK-SD-DOT-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-SD-DOT-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-SD-DOT-NEXT: umull v0.4s, v1.4h, v0.4h -; CHECK-SD-DOT-NEXT: addp v2.2s, v2.2s, v2.2s +; CHECK-SD-DOT-NEXT: umull v0.4s, v4.4h, v3.4h +; CHECK-SD-DOT-NEXT: addp v1.2s, v2.2s, v2.2s ; CHECK-SD-DOT-NEXT: fmov w9, s0 -; CHECK-SD-DOT-NEXT: fmov w8, s2 +; CHECK-SD-DOT-NEXT: fmov w8, s1 ; CHECK-SD-DOT-NEXT: add w0, w8, w9 ; CHECK-SD-DOT-NEXT: ret ; @@ -3534,21 +3534,21 @@ entry: define i64 @add_pair_v4i8_v4i64_sext(<4 x i8> %x, <4 x i8> %y) { ; CHECK-SD-LABEL: add_pair_v4i8_v4i64_sext: ; CHECK-SD: // %bb.0: // %entry -; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0 ; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-SD-NEXT: ushll v2.2d, v0.2s, #0 -; CHECK-SD-NEXT: ushll v3.2d, v1.2s, #0 +; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: ushll v2.2d, v1.2s, #0 +; CHECK-SD-NEXT: ushll v3.2d, v0.2s, #0 ; CHECK-SD-NEXT: ushll2 v0.2d, v0.4s, #0 ; CHECK-SD-NEXT: ushll2 v1.2d, v1.4s, #0 -; CHECK-SD-NEXT: shl v2.2d, v2.2d, #56 ; CHECK-SD-NEXT: shl v3.2d, v3.2d, #56 +; CHECK-SD-NEXT: shl v2.2d, v2.2d, #56 ; CHECK-SD-NEXT: shl v0.2d, v0.2d, #56 ; CHECK-SD-NEXT: shl v1.2d, v1.2d, #56 -; CHECK-SD-NEXT: sshr v2.2d, v2.2d, #56 ; CHECK-SD-NEXT: sshr v3.2d, v3.2d, #56 -; CHECK-SD-NEXT: ssra v2.2d, v0.2d, #56 -; CHECK-SD-NEXT: ssra v3.2d, v1.2d, #56 -; CHECK-SD-NEXT: add v0.2d, v2.2d, v3.2d +; CHECK-SD-NEXT: sshr v2.2d, v2.2d, #56 +; CHECK-SD-NEXT: ssra v3.2d, v0.2d, #56 +; CHECK-SD-NEXT: ssra v2.2d, v1.2d, #56 +; CHECK-SD-NEXT: add v0.2d, v3.2d, v2.2d ; CHECK-SD-NEXT: addp d0, v0.2d ; CHECK-SD-NEXT: fmov x0, d0 ; CHECK-SD-NEXT: ret @@ -3816,37 +3816,37 @@ define i16 @add_v24i8_v24i16_zext(<24 x i8> %x) { ; CHECK-SD-NEXT: ldr b1, [sp, #64] ; CHECK-SD-NEXT: add x8, sp, #72 ; CHECK-SD-NEXT: ldr b2, [sp] -; CHECK-SD-NEXT: add x9, sp, #8 +; CHECK-SD-NEXT: add x9, sp, #80 ; CHECK-SD-NEXT: ld1 { v1.b }[1], [x8] -; CHECK-SD-NEXT: add x8, sp, #80 +; CHECK-SD-NEXT: add x8, sp, #8 ; CHECK-SD-NEXT: mov v0.b[1], w1 -; CHECK-SD-NEXT: ld1 { v2.b }[1], [x9] -; CHECK-SD-NEXT: add x9, sp, #16 -; CHECK-SD-NEXT: ld1 { v1.b }[2], [x8] -; CHECK-SD-NEXT: add x8, sp, #88 -; CHECK-SD-NEXT: ld1 { v2.b }[2], [x9] -; CHECK-SD-NEXT: add x9, sp, #24 +; CHECK-SD-NEXT: ld1 { v2.b }[1], [x8] +; CHECK-SD-NEXT: add x8, sp, #16 +; CHECK-SD-NEXT: ld1 { v1.b }[2], [x9] +; CHECK-SD-NEXT: add x9, sp, #88 +; CHECK-SD-NEXT: ld1 { v2.b }[2], [x8] +; CHECK-SD-NEXT: add x8, sp, #24 ; CHECK-SD-NEXT: mov v0.b[2], w2 -; CHECK-SD-NEXT: ld1 { v1.b }[3], [x8] -; CHECK-SD-NEXT: add x8, sp, #96 -; CHECK-SD-NEXT: ld1 { v2.b }[3], [x9] -; CHECK-SD-NEXT: add x9, sp, #32 +; CHECK-SD-NEXT: ld1 { v1.b }[3], [x9] +; CHECK-SD-NEXT: add x9, sp, #96 +; CHECK-SD-NEXT: ld1 { v2.b }[3], [x8] +; CHECK-SD-NEXT: add x8, sp, #32 ; CHECK-SD-NEXT: mov v0.b[3], w3 -; CHECK-SD-NEXT: ld1 { v1.b }[4], [x8] -; CHECK-SD-NEXT: add x8, sp, #104 -; CHECK-SD-NEXT: ld1 { v2.b }[4], [x9] -; CHECK-SD-NEXT: add x9, sp, #40 -; CHECK-SD-NEXT: ld1 { v1.b }[5], [x8] -; CHECK-SD-NEXT: add x8, sp, #112 +; CHECK-SD-NEXT: ld1 { v1.b }[4], [x9] +; CHECK-SD-NEXT: add x9, sp, #104 +; CHECK-SD-NEXT: ld1 { v2.b }[4], [x8] +; CHECK-SD-NEXT: add x8, sp, #40 +; CHECK-SD-NEXT: ld1 { v1.b }[5], [x9] +; CHECK-SD-NEXT: add x9, sp, #112 ; CHECK-SD-NEXT: mov v0.b[4], w4 -; CHECK-SD-NEXT: ld1 { v2.b }[5], [x9] -; CHECK-SD-NEXT: add x9, sp, #48 -; CHECK-SD-NEXT: ld1 { v1.b }[6], [x8] -; CHECK-SD-NEXT: add x8, sp, #120 -; CHECK-SD-NEXT: ld1 { v2.b }[6], [x9] -; CHECK-SD-NEXT: mov v0.b[5], w5 -; CHECK-SD-NEXT: ld1 { v1.b }[7], [x8] +; CHECK-SD-NEXT: ld1 { v2.b }[5], [x8] +; CHECK-SD-NEXT: add x8, sp, #48 +; CHECK-SD-NEXT: ld1 { v1.b }[6], [x9] +; CHECK-SD-NEXT: add x9, sp, #120 +; CHECK-SD-NEXT: ld1 { v2.b }[6], [x8] ; CHECK-SD-NEXT: add x8, sp, #56 +; CHECK-SD-NEXT: mov v0.b[5], w5 +; CHECK-SD-NEXT: ld1 { v1.b }[7], [x9] ; CHECK-SD-NEXT: ld1 { v2.b }[7], [x8] ; CHECK-SD-NEXT: mov v0.b[6], w6 ; CHECK-SD-NEXT: mov v0.b[7], w7 @@ -3942,37 +3942,37 @@ define i16 @add_v24i8_v24i16_sext(<24 x i8> %x) { ; CHECK-SD-NEXT: ldr b1, [sp, #64] ; CHECK-SD-NEXT: add x8, sp, #72 ; CHECK-SD-NEXT: ldr b2, [sp] -; CHECK-SD-NEXT: add x9, sp, #8 +; CHECK-SD-NEXT: add x9, sp, #80 ; CHECK-SD-NEXT: ld1 { v1.b }[1], [x8] -; CHECK-SD-NEXT: add x8, sp, #80 +; CHECK-SD-NEXT: add x8, sp, #8 ; CHECK-SD-NEXT: mov v0.b[1], w1 -; CHECK-SD-NEXT: ld1 { v2.b }[1], [x9] -; CHECK-SD-NEXT: add x9, sp, #16 -; CHECK-SD-NEXT: ld1 { v1.b }[2], [x8] -; CHECK-SD-NEXT: add x8, sp, #88 -; CHECK-SD-NEXT: ld1 { v2.b }[2], [x9] -; CHECK-SD-NEXT: add x9, sp, #24 +; CHECK-SD-NEXT: ld1 { v2.b }[1], [x8] +; CHECK-SD-NEXT: add x8, sp, #16 +; CHECK-SD-NEXT: ld1 { v1.b }[2], [x9] +; CHECK-SD-NEXT: add x9, sp, #88 +; CHECK-SD-NEXT: ld1 { v2.b }[2], [x8] +; CHECK-SD-NEXT: add x8, sp, #24 ; CHECK-SD-NEXT: mov v0.b[2], w2 -; CHECK-SD-NEXT: ld1 { v1.b }[3], [x8] -; CHECK-SD-NEXT: add x8, sp, #96 -; CHECK-SD-NEXT: ld1 { v2.b }[3], [x9] -; CHECK-SD-NEXT: add x9, sp, #32 +; CHECK-SD-NEXT: ld1 { v1.b }[3], [x9] +; CHECK-SD-NEXT: add x9, sp, #96 +; CHECK-SD-NEXT: ld1 { v2.b }[3], [x8] +; CHECK-SD-NEXT: add x8, sp, #32 ; CHECK-SD-NEXT: mov v0.b[3], w3 -; CHECK-SD-NEXT: ld1 { v1.b }[4], [x8] -; CHECK-SD-NEXT: add x8, sp, #104 -; CHECK-SD-NEXT: ld1 { v2.b }[4], [x9] -; CHECK-SD-NEXT: add x9, sp, #40 -; CHECK-SD-NEXT: ld1 { v1.b }[5], [x8] -; CHECK-SD-NEXT: add x8, sp, #112 +; CHECK-SD-NEXT: ld1 { v1.b }[4], [x9] +; CHECK-SD-NEXT: add x9, sp, #104 +; CHECK-SD-NEXT: ld1 { v2.b }[4], [x8] +; CHECK-SD-NEXT: add x8, sp, #40 +; CHECK-SD-NEXT: ld1 { v1.b }[5], [x9] +; CHECK-SD-NEXT: add x9, sp, #112 ; CHECK-SD-NEXT: mov v0.b[4], w4 -; CHECK-SD-NEXT: ld1 { v2.b }[5], [x9] -; CHECK-SD-NEXT: add x9, sp, #48 -; CHECK-SD-NEXT: ld1 { v1.b }[6], [x8] -; CHECK-SD-NEXT: add x8, sp, #120 -; CHECK-SD-NEXT: ld1 { v2.b }[6], [x9] -; CHECK-SD-NEXT: mov v0.b[5], w5 -; CHECK-SD-NEXT: ld1 { v1.b }[7], [x8] +; CHECK-SD-NEXT: ld1 { v2.b }[5], [x8] +; CHECK-SD-NEXT: add x8, sp, #48 +; CHECK-SD-NEXT: ld1 { v1.b }[6], [x9] +; CHECK-SD-NEXT: add x9, sp, #120 +; CHECK-SD-NEXT: ld1 { v2.b }[6], [x8] ; CHECK-SD-NEXT: add x8, sp, #56 +; CHECK-SD-NEXT: mov v0.b[5], w5 +; CHECK-SD-NEXT: ld1 { v1.b }[7], [x9] ; CHECK-SD-NEXT: ld1 { v2.b }[7], [x8] ; CHECK-SD-NEXT: mov v0.b[6], w6 ; CHECK-SD-NEXT: mov v0.b[7], w7 @@ -4069,48 +4069,48 @@ define i32 @add_v24i8_v24i32_zext(<24 x i8> %x) { ; CHECK-SD-BASE-NEXT: ldr b1, [sp, #64] ; CHECK-SD-BASE-NEXT: add x8, sp, #72 ; CHECK-SD-BASE-NEXT: ldr b2, [sp] -; CHECK-SD-BASE-NEXT: add x9, sp, #8 +; CHECK-SD-BASE-NEXT: add x9, sp, #80 ; CHECK-SD-BASE-NEXT: ld1 { v1.b }[1], [x8] -; CHECK-SD-BASE-NEXT: add x8, sp, #80 +; CHECK-SD-BASE-NEXT: add x8, sp, #8 ; CHECK-SD-BASE-NEXT: mov v0.b[1], w1 -; CHECK-SD-BASE-NEXT: ld1 { v2.b }[1], [x9] -; CHECK-SD-BASE-NEXT: add x9, sp, #16 -; CHECK-SD-BASE-NEXT: ld1 { v1.b }[2], [x8] -; CHECK-SD-BASE-NEXT: add x8, sp, #88 -; CHECK-SD-BASE-NEXT: ld1 { v2.b }[2], [x9] -; CHECK-SD-BASE-NEXT: add x9, sp, #24 +; CHECK-SD-BASE-NEXT: ld1 { v2.b }[1], [x8] +; CHECK-SD-BASE-NEXT: add x8, sp, #16 +; CHECK-SD-BASE-NEXT: ld1 { v1.b }[2], [x9] +; CHECK-SD-BASE-NEXT: add x9, sp, #88 +; CHECK-SD-BASE-NEXT: ld1 { v2.b }[2], [x8] +; CHECK-SD-BASE-NEXT: add x8, sp, #24 ; CHECK-SD-BASE-NEXT: mov v0.b[2], w2 -; CHECK-SD-BASE-NEXT: ld1 { v1.b }[3], [x8] -; CHECK-SD-BASE-NEXT: add x8, sp, #96 -; CHECK-SD-BASE-NEXT: ld1 { v2.b }[3], [x9] -; CHECK-SD-BASE-NEXT: add x9, sp, #32 +; CHECK-SD-BASE-NEXT: ld1 { v1.b }[3], [x9] +; CHECK-SD-BASE-NEXT: add x9, sp, #96 +; CHECK-SD-BASE-NEXT: ld1 { v2.b }[3], [x8] +; CHECK-SD-BASE-NEXT: add x8, sp, #32 ; CHECK-SD-BASE-NEXT: mov v0.b[3], w3 -; CHECK-SD-BASE-NEXT: ld1 { v1.b }[4], [x8] -; CHECK-SD-BASE-NEXT: add x8, sp, #104 -; CHECK-SD-BASE-NEXT: ld1 { v2.b }[4], [x9] -; CHECK-SD-BASE-NEXT: add x9, sp, #40 -; CHECK-SD-BASE-NEXT: ld1 { v1.b }[5], [x8] -; CHECK-SD-BASE-NEXT: add x8, sp, #112 +; CHECK-SD-BASE-NEXT: ld1 { v1.b }[4], [x9] +; CHECK-SD-BASE-NEXT: add x9, sp, #104 +; CHECK-SD-BASE-NEXT: ld1 { v2.b }[4], [x8] +; CHECK-SD-BASE-NEXT: add x8, sp, #40 +; CHECK-SD-BASE-NEXT: ld1 { v1.b }[5], [x9] +; CHECK-SD-BASE-NEXT: add x9, sp, #112 ; CHECK-SD-BASE-NEXT: mov v0.b[4], w4 -; CHECK-SD-BASE-NEXT: ld1 { v2.b }[5], [x9] -; CHECK-SD-BASE-NEXT: add x9, sp, #48 -; CHECK-SD-BASE-NEXT: ld1 { v1.b }[6], [x8] -; CHECK-SD-BASE-NEXT: add x8, sp, #120 -; CHECK-SD-BASE-NEXT: ld1 { v2.b }[6], [x9] -; CHECK-SD-BASE-NEXT: mov v0.b[5], w5 -; CHECK-SD-BASE-NEXT: ld1 { v1.b }[7], [x8] +; CHECK-SD-BASE-NEXT: ld1 { v2.b }[5], [x8] +; CHECK-SD-BASE-NEXT: add x8, sp, #48 +; CHECK-SD-BASE-NEXT: ld1 { v1.b }[6], [x9] +; CHECK-SD-BASE-NEXT: add x9, sp, #120 +; CHECK-SD-BASE-NEXT: ld1 { v2.b }[6], [x8] ; CHECK-SD-BASE-NEXT: add x8, sp, #56 +; CHECK-SD-BASE-NEXT: mov v0.b[5], w5 +; CHECK-SD-BASE-NEXT: ld1 { v1.b }[7], [x9] ; CHECK-SD-BASE-NEXT: ld1 { v2.b }[7], [x8] ; CHECK-SD-BASE-NEXT: mov v0.b[6], w6 ; CHECK-SD-BASE-NEXT: ushll v1.8h, v1.8b, #0 +; CHECK-SD-BASE-NEXT: ushll v2.8h, v2.8b, #0 ; CHECK-SD-BASE-NEXT: mov v0.b[7], w7 ; CHECK-SD-BASE-NEXT: ushll v0.8h, v0.8b, #0 ; CHECK-SD-BASE-NEXT: uaddl2 v3.4s, v0.8h, v1.8h ; CHECK-SD-BASE-NEXT: uaddl v0.4s, v0.4h, v1.4h -; CHECK-SD-BASE-NEXT: ushll v1.8h, v2.8b, #0 -; CHECK-SD-BASE-NEXT: uaddw2 v2.4s, v3.4s, v1.8h -; CHECK-SD-BASE-NEXT: uaddw v0.4s, v0.4s, v1.4h -; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v2.4s +; CHECK-SD-BASE-NEXT: uaddw2 v1.4s, v3.4s, v2.8h +; CHECK-SD-BASE-NEXT: uaddw v0.4s, v0.4s, v2.4h +; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-SD-BASE-NEXT: addv s0, v0.4s ; CHECK-SD-BASE-NEXT: fmov w0, s0 ; CHECK-SD-BASE-NEXT: ret @@ -4147,9 +4147,9 @@ define i32 @add_v24i8_v24i32_zext(<24 x i8> %x) { ; CHECK-SD-DOT-NEXT: udot v4.2s, v1.8b, v5.8b ; CHECK-SD-DOT-NEXT: mov v0.b[7], w7 ; CHECK-SD-DOT-NEXT: addp v1.2s, v4.2s, v4.2s +; CHECK-SD-DOT-NEXT: fmov w9, s1 ; CHECK-SD-DOT-NEXT: ld1 { v0.b }[8], [x8] ; CHECK-SD-DOT-NEXT: add x8, sp, #8 -; CHECK-SD-DOT-NEXT: fmov w9, s1 ; CHECK-SD-DOT-NEXT: ld1 { v0.b }[9], [x8] ; CHECK-SD-DOT-NEXT: add x8, sp, #16 ; CHECK-SD-DOT-NEXT: ld1 { v0.b }[10], [x8] @@ -4342,48 +4342,48 @@ define i32 @add_v24i8_v24i32_sext(<24 x i8> %x) { ; CHECK-SD-BASE-NEXT: ldr b1, [sp, #64] ; CHECK-SD-BASE-NEXT: add x8, sp, #72 ; CHECK-SD-BASE-NEXT: ldr b2, [sp] -; CHECK-SD-BASE-NEXT: add x9, sp, #8 +; CHECK-SD-BASE-NEXT: add x9, sp, #80 ; CHECK-SD-BASE-NEXT: ld1 { v1.b }[1], [x8] -; CHECK-SD-BASE-NEXT: add x8, sp, #80 +; CHECK-SD-BASE-NEXT: add x8, sp, #8 ; CHECK-SD-BASE-NEXT: mov v0.b[1], w1 -; CHECK-SD-BASE-NEXT: ld1 { v2.b }[1], [x9] -; CHECK-SD-BASE-NEXT: add x9, sp, #16 -; CHECK-SD-BASE-NEXT: ld1 { v1.b }[2], [x8] -; CHECK-SD-BASE-NEXT: add x8, sp, #88 -; CHECK-SD-BASE-NEXT: ld1 { v2.b }[2], [x9] -; CHECK-SD-BASE-NEXT: add x9, sp, #24 +; CHECK-SD-BASE-NEXT: ld1 { v2.b }[1], [x8] +; CHECK-SD-BASE-NEXT: add x8, sp, #16 +; CHECK-SD-BASE-NEXT: ld1 { v1.b }[2], [x9] +; CHECK-SD-BASE-NEXT: add x9, sp, #88 +; CHECK-SD-BASE-NEXT: ld1 { v2.b }[2], [x8] +; CHECK-SD-BASE-NEXT: add x8, sp, #24 ; CHECK-SD-BASE-NEXT: mov v0.b[2], w2 -; CHECK-SD-BASE-NEXT: ld1 { v1.b }[3], [x8] -; CHECK-SD-BASE-NEXT: add x8, sp, #96 -; CHECK-SD-BASE-NEXT: ld1 { v2.b }[3], [x9] -; CHECK-SD-BASE-NEXT: add x9, sp, #32 +; CHECK-SD-BASE-NEXT: ld1 { v1.b }[3], [x9] +; CHECK-SD-BASE-NEXT: add x9, sp, #96 +; CHECK-SD-BASE-NEXT: ld1 { v2.b }[3], [x8] +; CHECK-SD-BASE-NEXT: add x8, sp, #32 ; CHECK-SD-BASE-NEXT: mov v0.b[3], w3 -; CHECK-SD-BASE-NEXT: ld1 { v1.b }[4], [x8] -; CHECK-SD-BASE-NEXT: add x8, sp, #104 -; CHECK-SD-BASE-NEXT: ld1 { v2.b }[4], [x9] -; CHECK-SD-BASE-NEXT: add x9, sp, #40 -; CHECK-SD-BASE-NEXT: ld1 { v1.b }[5], [x8] -; CHECK-SD-BASE-NEXT: add x8, sp, #112 +; CHECK-SD-BASE-NEXT: ld1 { v1.b }[4], [x9] +; CHECK-SD-BASE-NEXT: add x9, sp, #104 +; CHECK-SD-BASE-NEXT: ld1 { v2.b }[4], [x8] +; CHECK-SD-BASE-NEXT: add x8, sp, #40 +; CHECK-SD-BASE-NEXT: ld1 { v1.b }[5], [x9] +; CHECK-SD-BASE-NEXT: add x9, sp, #112 ; CHECK-SD-BASE-NEXT: mov v0.b[4], w4 -; CHECK-SD-BASE-NEXT: ld1 { v2.b }[5], [x9] -; CHECK-SD-BASE-NEXT: add x9, sp, #48 -; CHECK-SD-BASE-NEXT: ld1 { v1.b }[6], [x8] -; CHECK-SD-BASE-NEXT: add x8, sp, #120 -; CHECK-SD-BASE-NEXT: ld1 { v2.b }[6], [x9] -; CHECK-SD-BASE-NEXT: mov v0.b[5], w5 -; CHECK-SD-BASE-NEXT: ld1 { v1.b }[7], [x8] +; CHECK-SD-BASE-NEXT: ld1 { v2.b }[5], [x8] +; CHECK-SD-BASE-NEXT: add x8, sp, #48 +; CHECK-SD-BASE-NEXT: ld1 { v1.b }[6], [x9] +; CHECK-SD-BASE-NEXT: add x9, sp, #120 +; CHECK-SD-BASE-NEXT: ld1 { v2.b }[6], [x8] ; CHECK-SD-BASE-NEXT: add x8, sp, #56 +; CHECK-SD-BASE-NEXT: mov v0.b[5], w5 +; CHECK-SD-BASE-NEXT: ld1 { v1.b }[7], [x9] ; CHECK-SD-BASE-NEXT: ld1 { v2.b }[7], [x8] ; CHECK-SD-BASE-NEXT: mov v0.b[6], w6 ; CHECK-SD-BASE-NEXT: sshll v1.8h, v1.8b, #0 +; CHECK-SD-BASE-NEXT: sshll v2.8h, v2.8b, #0 ; CHECK-SD-BASE-NEXT: mov v0.b[7], w7 ; CHECK-SD-BASE-NEXT: sshll v0.8h, v0.8b, #0 ; CHECK-SD-BASE-NEXT: saddl2 v3.4s, v0.8h, v1.8h ; CHECK-SD-BASE-NEXT: saddl v0.4s, v0.4h, v1.4h -; CHECK-SD-BASE-NEXT: sshll v1.8h, v2.8b, #0 -; CHECK-SD-BASE-NEXT: saddw2 v2.4s, v3.4s, v1.8h -; CHECK-SD-BASE-NEXT: saddw v0.4s, v0.4s, v1.4h -; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v2.4s +; CHECK-SD-BASE-NEXT: saddw2 v1.4s, v3.4s, v2.8h +; CHECK-SD-BASE-NEXT: saddw v0.4s, v0.4s, v2.4h +; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-SD-BASE-NEXT: addv s0, v0.4s ; CHECK-SD-BASE-NEXT: fmov w0, s0 ; CHECK-SD-BASE-NEXT: ret @@ -4420,9 +4420,9 @@ define i32 @add_v24i8_v24i32_sext(<24 x i8> %x) { ; CHECK-SD-DOT-NEXT: sdot v4.2s, v1.8b, v5.8b ; CHECK-SD-DOT-NEXT: mov v0.b[7], w7 ; CHECK-SD-DOT-NEXT: addp v1.2s, v4.2s, v4.2s +; CHECK-SD-DOT-NEXT: fmov w9, s1 ; CHECK-SD-DOT-NEXT: ld1 { v0.b }[8], [x8] ; CHECK-SD-DOT-NEXT: add x8, sp, #8 -; CHECK-SD-DOT-NEXT: fmov w9, s1 ; CHECK-SD-DOT-NEXT: ld1 { v0.b }[9], [x8] ; CHECK-SD-DOT-NEXT: add x8, sp, #16 ; CHECK-SD-DOT-NEXT: ld1 { v0.b }[10], [x8] @@ -4611,23 +4611,23 @@ entry: define i32 @full(ptr %p1, i32 noundef %s1, ptr %p2, i32 noundef %s2) { ; CHECK-SD-BASE-LABEL: full: ; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: ldr d0, [x2] -; CHECK-SD-BASE-NEXT: ldr d1, [x0] ; CHECK-SD-BASE-NEXT: // kill: def $w3 killed $w3 def $x3 ; CHECK-SD-BASE-NEXT: // kill: def $w1 killed $w1 def $x1 ; CHECK-SD-BASE-NEXT: sxtw x8, w3 ; CHECK-SD-BASE-NEXT: sxtw x9, w1 -; CHECK-SD-BASE-NEXT: uabdl v0.8h, v1.8b, v0.8b -; CHECK-SD-BASE-NEXT: add x11, x2, x8 +; CHECK-SD-BASE-NEXT: ldr d0, [x0] +; CHECK-SD-BASE-NEXT: ldr d1, [x2] ; CHECK-SD-BASE-NEXT: add x10, x0, x9 -; CHECK-SD-BASE-NEXT: ldr d2, [x11] -; CHECK-SD-BASE-NEXT: add x11, x11, x8 +; CHECK-SD-BASE-NEXT: add x11, x2, x8 +; CHECK-SD-BASE-NEXT: uabdl v0.8h, v0.8b, v1.8b ; CHECK-SD-BASE-NEXT: ldr d1, [x10] +; CHECK-SD-BASE-NEXT: ldr d2, [x11] ; CHECK-SD-BASE-NEXT: add x10, x10, x9 -; CHECK-SD-BASE-NEXT: uaddlp v0.4s, v0.8h +; CHECK-SD-BASE-NEXT: add x11, x11, x8 ; CHECK-SD-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b ; CHECK-SD-BASE-NEXT: ldr d2, [x11] ; CHECK-SD-BASE-NEXT: add x11, x11, x8 +; CHECK-SD-BASE-NEXT: uaddlp v0.4s, v0.8h ; CHECK-SD-BASE-NEXT: uadalp v0.4s, v1.8h ; CHECK-SD-BASE-NEXT: ldr d1, [x10] ; CHECK-SD-BASE-NEXT: add x10, x10, x9 @@ -4723,98 +4723,98 @@ define i32 @full(ptr %p1, i32 noundef %s1, ptr %p2, i32 noundef %s2) { ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: // kill: def $w1 killed $w1 def $x1 ; CHECK-GI-NEXT: // kill: def $w3 killed $w3 def $x3 -; CHECK-GI-NEXT: sxtw x8, w1 -; CHECK-GI-NEXT: sxtw x9, w3 +; CHECK-GI-NEXT: sxtw x8, w3 +; CHECK-GI-NEXT: sxtw x9, w1 ; CHECK-GI-NEXT: ldr d0, [x0] ; CHECK-GI-NEXT: ldr d1, [x2] -; CHECK-GI-NEXT: add x10, x0, x8 -; CHECK-GI-NEXT: add x11, x2, x9 +; CHECK-GI-NEXT: add x10, x0, x9 +; CHECK-GI-NEXT: add x11, x2, x8 ; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0 ; CHECK-GI-NEXT: ldr d2, [x10] -; CHECK-GI-NEXT: ldr d3, [x11] +; CHECK-GI-NEXT: add x10, x10, x9 +; CHECK-GI-NEXT: add x12, x11, x8 ; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-GI-NEXT: add x10, x10, x8 -; CHECK-GI-NEXT: add x11, x11, x9 +; CHECK-GI-NEXT: ldr d3, [x11] +; CHECK-GI-NEXT: ldr d4, [x10] +; CHECK-GI-NEXT: ldr d5, [x12] +; CHECK-GI-NEXT: add x10, x10, x9 +; CHECK-GI-NEXT: add x11, x12, x8 ; CHECK-GI-NEXT: ushll v2.8h, v2.8b, #0 ; CHECK-GI-NEXT: ushll v3.8h, v3.8b, #0 -; CHECK-GI-NEXT: ldr d4, [x10] -; CHECK-GI-NEXT: ldr d5, [x11] -; CHECK-GI-NEXT: add x10, x10, x8 -; CHECK-GI-NEXT: add x11, x11, x9 +; CHECK-GI-NEXT: ushll v4.8h, v4.8b, #0 +; CHECK-GI-NEXT: ushll v5.8h, v5.8b, #0 ; CHECK-GI-NEXT: uabdl v6.4s, v0.4h, v1.4h ; CHECK-GI-NEXT: uabdl2 v0.4s, v0.8h, v1.8h ; CHECK-GI-NEXT: ldr d1, [x10] -; CHECK-GI-NEXT: ushll v4.8h, v4.8b, #0 -; CHECK-GI-NEXT: ushll v5.8h, v5.8b, #0 ; CHECK-GI-NEXT: ldr d7, [x11] +; CHECK-GI-NEXT: add x10, x10, x9 +; CHECK-GI-NEXT: add x11, x11, x8 ; CHECK-GI-NEXT: uabdl v16.4s, v2.4h, v3.4h ; CHECK-GI-NEXT: uabdl2 v2.4s, v2.8h, v3.8h -; CHECK-GI-NEXT: ushll v3.8h, v1.8b, #0 -; CHECK-GI-NEXT: ushll v7.8h, v7.8b, #0 -; CHECK-GI-NEXT: add x10, x10, x8 -; CHECK-GI-NEXT: add x11, x11, x9 -; CHECK-GI-NEXT: uabdl v1.4s, v4.4h, v5.4h +; CHECK-GI-NEXT: uabdl v3.4s, v4.4h, v5.4h ; CHECK-GI-NEXT: uabdl2 v4.4s, v4.8h, v5.8h +; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #0 +; CHECK-GI-NEXT: ushll v7.8h, v7.8b, #0 ; CHECK-GI-NEXT: ldr d5, [x10] -; CHECK-GI-NEXT: add v2.4s, v16.4s, v2.4s -; CHECK-GI-NEXT: ldr d16, [x11] +; CHECK-GI-NEXT: ldr d17, [x11] +; CHECK-GI-NEXT: add x10, x10, x9 +; CHECK-GI-NEXT: add x11, x11, x8 ; CHECK-GI-NEXT: add v0.4s, v6.4s, v0.4s -; CHECK-GI-NEXT: uabdl v6.4s, v3.4h, v7.4h -; CHECK-GI-NEXT: uabdl2 v3.4s, v3.8h, v7.8h ; CHECK-GI-NEXT: ushll v5.8h, v5.8b, #0 -; CHECK-GI-NEXT: add x10, x10, x8 -; CHECK-GI-NEXT: ushll v7.8h, v16.8b, #0 -; CHECK-GI-NEXT: add x11, x11, x9 -; CHECK-GI-NEXT: ldr d16, [x10] -; CHECK-GI-NEXT: ldr d17, [x11] -; CHECK-GI-NEXT: add v1.4s, v1.4s, v4.4s -; CHECK-GI-NEXT: add x10, x10, x8 -; CHECK-GI-NEXT: add x11, x11, x9 -; CHECK-GI-NEXT: add v3.4s, v6.4s, v3.4s -; CHECK-GI-NEXT: ushll v16.8h, v16.8b, #0 ; CHECK-GI-NEXT: ushll v17.8h, v17.8b, #0 -; CHECK-GI-NEXT: uabdl v22.4s, v5.4h, v7.4h -; CHECK-GI-NEXT: uabdl2 v5.4s, v5.8h, v7.8h +; CHECK-GI-NEXT: add v2.4s, v16.4s, v2.4s +; CHECK-GI-NEXT: add v3.4s, v3.4s, v4.4s +; CHECK-GI-NEXT: uabdl v4.4s, v1.4h, v7.4h +; CHECK-GI-NEXT: uabdl2 v1.4s, v1.8h, v7.8h +; CHECK-GI-NEXT: ldr d7, [x10] +; CHECK-GI-NEXT: ldr d16, [x11] +; CHECK-GI-NEXT: add x10, x10, x9 +; CHECK-GI-NEXT: add x11, x11, x8 ; CHECK-GI-NEXT: ldr d18, [x10] +; CHECK-GI-NEXT: ldr d20, [x10, x9] ; CHECK-GI-NEXT: ldr d19, [x11] -; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: ldr d21, [x11, x8] +; CHECK-GI-NEXT: uabdl v6.4s, v5.4h, v17.4h +; CHECK-GI-NEXT: ushll v7.8h, v7.8b, #0 +; CHECK-GI-NEXT: ushll v16.8h, v16.8b, #0 +; CHECK-GI-NEXT: uabdl2 v5.4s, v5.8h, v17.8h +; CHECK-GI-NEXT: ushll v17.8h, v18.8b, #0 +; CHECK-GI-NEXT: ushll v18.8h, v19.8b, #0 +; CHECK-GI-NEXT: add v1.4s, v4.4s, v1.4s +; CHECK-GI-NEXT: ushll v4.8h, v20.8b, #0 +; CHECK-GI-NEXT: ushll v19.8h, v21.8b, #0 ; CHECK-GI-NEXT: addv s2, v2.4s -; CHECK-GI-NEXT: addv s1, v1.4s -; CHECK-GI-NEXT: ushll v18.8h, v18.8b, #0 -; CHECK-GI-NEXT: ushll v19.8h, v19.8b, #0 -; CHECK-GI-NEXT: uabdl v4.4s, v16.4h, v17.4h -; CHECK-GI-NEXT: uabdl2 v16.4s, v16.8h, v17.8h -; CHECK-GI-NEXT: add v5.4s, v22.4s, v5.4s -; CHECK-GI-NEXT: ldr d20, [x10, x8] -; CHECK-GI-NEXT: ldr d21, [x11, x9] +; CHECK-GI-NEXT: addv s0, v0.4s ; CHECK-GI-NEXT: addv s3, v3.4s +; CHECK-GI-NEXT: uabdl v20.4s, v7.4h, v16.4h +; CHECK-GI-NEXT: uabdl2 v7.4s, v7.8h, v16.8h +; CHECK-GI-NEXT: add v5.4s, v6.4s, v5.4s +; CHECK-GI-NEXT: uabdl v6.4s, v17.4h, v18.4h +; CHECK-GI-NEXT: uabdl2 v16.4s, v17.8h, v18.8h +; CHECK-GI-NEXT: uabdl v17.4s, v4.4h, v19.4h +; CHECK-GI-NEXT: uabdl2 v4.4s, v4.8h, v19.8h ; CHECK-GI-NEXT: fmov w8, s2 +; CHECK-GI-NEXT: addv s1, v1.4s ; CHECK-GI-NEXT: fmov w9, s0 -; CHECK-GI-NEXT: ushll v7.8h, v20.8b, #0 -; CHECK-GI-NEXT: ushll v20.8h, v21.8b, #0 -; CHECK-GI-NEXT: uabdl v6.4s, v18.4h, v19.4h -; CHECK-GI-NEXT: uabdl2 v17.4s, v18.8h, v19.8h -; CHECK-GI-NEXT: add v4.4s, v4.4s, v16.4s -; CHECK-GI-NEXT: addv s5, v5.4s -; CHECK-GI-NEXT: fmov w10, s1 +; CHECK-GI-NEXT: fmov w10, s3 +; CHECK-GI-NEXT: add v7.4s, v20.4s, v7.4s +; CHECK-GI-NEXT: add v0.4s, v17.4s, v4.4s +; CHECK-GI-NEXT: addv s4, v5.4s +; CHECK-GI-NEXT: add v2.4s, v6.4s, v16.4s ; CHECK-GI-NEXT: add w8, w8, w9 -; CHECK-GI-NEXT: fmov w9, s3 -; CHECK-GI-NEXT: uabdl v18.4s, v7.4h, v20.4h -; CHECK-GI-NEXT: uabdl2 v7.4s, v7.8h, v20.8h -; CHECK-GI-NEXT: add v6.4s, v6.4s, v17.4s +; CHECK-GI-NEXT: fmov w9, s1 ; CHECK-GI-NEXT: add w8, w10, w8 -; CHECK-GI-NEXT: addv s0, v4.4s +; CHECK-GI-NEXT: addv s3, v7.4s +; CHECK-GI-NEXT: addv s1, v2.4s +; CHECK-GI-NEXT: addv s0, v0.4s ; CHECK-GI-NEXT: add w8, w9, w8 -; CHECK-GI-NEXT: fmov w9, s5 -; CHECK-GI-NEXT: add v7.4s, v18.4s, v7.4s -; CHECK-GI-NEXT: addv s1, v6.4s +; CHECK-GI-NEXT: fmov w9, s4 ; CHECK-GI-NEXT: add w8, w9, w8 -; CHECK-GI-NEXT: fmov w9, s0 -; CHECK-GI-NEXT: addv s2, v7.4s +; CHECK-GI-NEXT: fmov w9, s3 ; CHECK-GI-NEXT: add w8, w9, w8 ; CHECK-GI-NEXT: fmov w9, s1 ; CHECK-GI-NEXT: add w8, w9, w8 -; CHECK-GI-NEXT: fmov w9, s2 +; CHECK-GI-NEXT: fmov w9, s0 ; CHECK-GI-NEXT: add w0, w9, w8 ; CHECK-GI-NEXT: ret entry: diff --git a/llvm/test/CodeGen/AArch64/vector-fcopysign.ll b/llvm/test/CodeGen/AArch64/vector-fcopysign.ll index de26676b5c73..063b23275c61 100644 --- a/llvm/test/CodeGen/AArch64/vector-fcopysign.ll +++ b/llvm/test/CodeGen/AArch64/vector-fcopysign.ll @@ -264,15 +264,15 @@ define <4 x bfloat> @test_copysign_v4bf16_v4bf16(<4 x bfloat> %a, <4 x bfloat> % define <4 x bfloat> @test_copysign_v4bf16_v4f32(<4 x bfloat> %a, <4 x float> %b) #0 { ; CHECK-LABEL: test_copysign_v4bf16_v4f32: ; CHECK: ; %bb.0: -; CHECK-NEXT: movi.4s v2, #127, msl #8 -; CHECK-NEXT: movi.4s v3, #1 +; CHECK-NEXT: movi.4s v2, #1 +; CHECK-NEXT: movi.4s v3, #127, msl #8 ; CHECK-NEXT: ushr.4s v4, v1, #16 -; CHECK-NEXT: add.4s v2, v1, v2 -; CHECK-NEXT: and.16b v3, v4, v3 -; CHECK-NEXT: add.4s v2, v3, v2 -; CHECK-NEXT: fcmeq.4s v3, v1, v1 +; CHECK-NEXT: and.16b v2, v4, v2 +; CHECK-NEXT: add.4s v3, v1, v3 +; CHECK-NEXT: fcmeq.4s v4, v1, v1 ; CHECK-NEXT: orr.4s v1, #64, lsl #16 -; CHECK-NEXT: bit.16b v1, v2, v3 +; CHECK-NEXT: add.4s v2, v2, v3 +; CHECK-NEXT: bit.16b v1, v2, v4 ; CHECK-NEXT: mvni.4h v2, #128, lsl #8 ; CHECK-NEXT: shrn.4h v1, v1, #16 ; CHECK-NEXT: bif.8b v0, v1, v2 @@ -286,16 +286,16 @@ define <4 x bfloat> @test_copysign_v4bf16_v4f64(<4 x bfloat> %a, <4 x double> %b ; CHECK-LABEL: test_copysign_v4bf16_v4f64: ; CHECK: ; %bb.0: ; CHECK-NEXT: fcvtxn v1.2s, v1.2d -; CHECK-NEXT: movi.4s v3, #1 +; CHECK-NEXT: movi.4s v3, #127, msl #8 ; CHECK-NEXT: fcvtxn2 v1.4s, v2.2d -; CHECK-NEXT: movi.4s v2, #127, msl #8 +; CHECK-NEXT: movi.4s v2, #1 ; CHECK-NEXT: ushr.4s v4, v1, #16 -; CHECK-NEXT: add.4s v2, v1, v2 -; CHECK-NEXT: and.16b v3, v4, v3 -; CHECK-NEXT: add.4s v2, v3, v2 -; CHECK-NEXT: fcmeq.4s v3, v1, v1 +; CHECK-NEXT: add.4s v3, v1, v3 +; CHECK-NEXT: and.16b v2, v4, v2 +; CHECK-NEXT: fcmeq.4s v4, v1, v1 ; CHECK-NEXT: orr.4s v1, #64, lsl #16 -; CHECK-NEXT: bit.16b v1, v2, v3 +; CHECK-NEXT: add.4s v2, v2, v3 +; CHECK-NEXT: bit.16b v1, v2, v4 ; CHECK-NEXT: mvni.4h v2, #128, lsl #8 ; CHECK-NEXT: shrn.4h v1, v1, #16 ; CHECK-NEXT: bif.8b v0, v1, v2 @@ -322,22 +322,22 @@ define <8 x bfloat> @test_copysign_v8bf16_v8bf16(<8 x bfloat> %a, <8 x bfloat> % define <8 x bfloat> @test_copysign_v8bf16_v8f32(<8 x bfloat> %a, <8 x float> %b) #0 { ; CHECK-LABEL: test_copysign_v8bf16_v8f32: ; CHECK: ; %bb.0: -; CHECK-NEXT: movi.4s v3, #127, msl #8 -; CHECK-NEXT: movi.4s v4, #1 +; CHECK-NEXT: movi.4s v3, #1 +; CHECK-NEXT: movi.4s v4, #127, msl #8 ; CHECK-NEXT: ushr.4s v5, v2, #16 ; CHECK-NEXT: ushr.4s v6, v1, #16 -; CHECK-NEXT: add.4s v7, v2, v3 -; CHECK-NEXT: add.4s v3, v1, v3 -; CHECK-NEXT: and.16b v5, v5, v4 -; CHECK-NEXT: and.16b v4, v6, v4 +; CHECK-NEXT: and.16b v5, v5, v3 +; CHECK-NEXT: add.4s v7, v2, v4 +; CHECK-NEXT: and.16b v3, v6, v3 +; CHECK-NEXT: add.4s v4, v1, v4 ; CHECK-NEXT: fcmeq.4s v6, v2, v2 ; CHECK-NEXT: orr.4s v2, #64, lsl #16 ; CHECK-NEXT: add.4s v5, v5, v7 -; CHECK-NEXT: add.4s v3, v4, v3 -; CHECK-NEXT: fcmeq.4s v4, v1, v1 +; CHECK-NEXT: fcmeq.4s v7, v1, v1 ; CHECK-NEXT: orr.4s v1, #64, lsl #16 +; CHECK-NEXT: add.4s v3, v3, v4 ; CHECK-NEXT: bit.16b v2, v5, v6 -; CHECK-NEXT: bit.16b v1, v3, v4 +; CHECK-NEXT: bit.16b v1, v3, v7 ; CHECK-NEXT: uzp2.8h v1, v1, v2 ; CHECK-NEXT: mvni.8h v2, #128, lsl #8 ; CHECK-NEXT: bif.16b v0, v1, v2 diff --git a/llvm/test/CodeGen/AArch64/vector-gep.ll b/llvm/test/CodeGen/AArch64/vector-gep.ll index 30317dce85e6..c7858416e179 100644 --- a/llvm/test/CodeGen/AArch64/vector-gep.ll +++ b/llvm/test/CodeGen/AArch64/vector-gep.ll @@ -13,11 +13,11 @@ define <2 x ptr> @vector_gep(<2 x ptr> %0) { ; CHECK: ; %bb.0: ; %entry ; CHECK-NEXT: Lloh0: ; CHECK-NEXT: adrp x8, lCPI0_0@PAGE +; CHECK-NEXT: movi v2.2d, #0x000000ffffffff ; CHECK-NEXT: Lloh1: ; CHECK-NEXT: ldr q1, [x8, lCPI0_0@PAGEOFF] ; CHECK-NEXT: add v0.2d, v0.2d, v1.2d -; CHECK-NEXT: movi v1.2d, #0x000000ffffffff -; CHECK-NEXT: and v0.16b, v0.16b, v1.16b +; CHECK-NEXT: and v0.16b, v0.16b, v2.16b ; CHECK-NEXT: ret ; CHECK-NEXT: .loh AdrpLdr Lloh0, Lloh1 entry: diff --git a/llvm/test/CodeGen/AArch64/vselect-constants.ll b/llvm/test/CodeGen/AArch64/vselect-constants.ll index a32147eebd75..5e6ff1e0740c 100644 --- a/llvm/test/CodeGen/AArch64/vselect-constants.ll +++ b/llvm/test/CodeGen/AArch64/vselect-constants.ll @@ -370,10 +370,9 @@ define @signbit_mask_xor_nxv16i8( %a, %a, zeroinitializer %xor = xor %a, %b diff --git a/llvm/test/CodeGen/AArch64/zext-to-tbl.ll b/llvm/test/CodeGen/AArch64/zext-to-tbl.ll index 08ad34c7b03b..599bd811d7d5 100644 --- a/llvm/test/CodeGen/AArch64/zext-to-tbl.ll +++ b/llvm/test/CodeGen/AArch64/zext-to-tbl.ll @@ -1697,14 +1697,14 @@ define void @zext_v8i8_to_v8i64_with_add_in_sequence_in_loop(ptr %src, ptr %dst) ; CHECK-NEXT: ldp q18, q16, [x10, #96] ; CHECK-NEXT: uaddw.2d v2, v17, v2 ; CHECK-NEXT: stp q4, q5, [x10, #32] -; CHECK-NEXT: ldp q17, q5, [x10, #64] -; CHECK-NEXT: uaddw2.2d v16, v16, v7 +; CHECK-NEXT: uaddw2.2d v5, v16, v7 +; CHECK-NEXT: ldp q16, q4, [x10, #64] ; CHECK-NEXT: uaddw.2d v7, v18, v7 ; CHECK-NEXT: stp q2, q6, [x10] -; CHECK-NEXT: uaddw2.2d v4, v5, v3 -; CHECK-NEXT: uaddw.2d v3, v17, v3 -; CHECK-NEXT: stp q7, q16, [x10, #96] -; CHECK-NEXT: stp q3, q4, [x10, #64] +; CHECK-NEXT: uaddw2.2d v4, v4, v3 +; CHECK-NEXT: uaddw.2d v2, v16, v3 +; CHECK-NEXT: stp q7, q5, [x10, #96] +; CHECK-NEXT: stp q2, q4, [x10, #64] ; CHECK-NEXT: b.ne LBB17_1 ; CHECK-NEXT: ; %bb.2: ; %exit ; CHECK-NEXT: ret @@ -1729,15 +1729,15 @@ define void @zext_v8i8_to_v8i64_with_add_in_sequence_in_loop(ptr %src, ptr %dst) ; CHECK-BE-NEXT: ld1 { v3.8b }, [x10] ; CHECK-BE-NEXT: add x10, x1, x8 ; CHECK-BE-NEXT: add x8, x8, #128 -; CHECK-BE-NEXT: add x15, x10, #96 ; CHECK-BE-NEXT: add x11, x10, #32 ; CHECK-BE-NEXT: add x14, x10, #64 +; CHECK-BE-NEXT: add x15, x10, #96 ; CHECK-BE-NEXT: tbl v4.16b, { v2.16b }, v1.16b ; CHECK-BE-NEXT: tbl v2.16b, { v2.16b }, v0.16b -; CHECK-BE-NEXT: ld1 { v16.2d }, [x15] -; CHECK-BE-NEXT: tbl v5.16b, { v3.16b }, v1.16b +; CHECK-BE-NEXT: ld1 { v5.2d }, [x10] +; CHECK-BE-NEXT: tbl v6.16b, { v3.16b }, v1.16b ; CHECK-BE-NEXT: tbl v3.16b, { v3.16b }, v0.16b -; CHECK-BE-NEXT: ld1 { v6.2d }, [x10] +; CHECK-BE-NEXT: ld1 { v16.2d }, [x15] ; CHECK-BE-NEXT: ld1 { v19.2d }, [x14] ; CHECK-BE-NEXT: ld1 { v21.2d }, [x11] ; CHECK-BE-NEXT: add x12, x10, #48 @@ -1747,11 +1747,12 @@ define void @zext_v8i8_to_v8i64_with_add_in_sequence_in_loop(ptr %src, ptr %dst) ; CHECK-BE-NEXT: rev32 v7.8b, v4.8b ; CHECK-BE-NEXT: ext v4.16b, v4.16b, v4.16b, #8 ; CHECK-BE-NEXT: rev32 v17.8b, v2.8b -; CHECK-BE-NEXT: ext v18.16b, v5.16b, v5.16b, #8 +; CHECK-BE-NEXT: ext v18.16b, v6.16b, v6.16b, #8 ; CHECK-BE-NEXT: ext v20.16b, v3.16b, v3.16b, #8 ; CHECK-BE-NEXT: ext v2.16b, v2.16b, v2.16b, #8 -; CHECK-BE-NEXT: rev32 v5.8b, v5.8b +; CHECK-BE-NEXT: rev32 v6.8b, v6.8b ; CHECK-BE-NEXT: rev32 v3.8b, v3.8b +; CHECK-BE-NEXT: ld1 { v22.2d }, [x12] ; CHECK-BE-NEXT: cmp x8, #1024 ; CHECK-BE-NEXT: rev32 v4.8b, v4.8b ; CHECK-BE-NEXT: uaddw v7.2d, v16.2d, v7.2s @@ -1760,22 +1761,21 @@ define void @zext_v8i8_to_v8i64_with_add_in_sequence_in_loop(ptr %src, ptr %dst) ; CHECK-BE-NEXT: rev32 v20.8b, v20.8b ; CHECK-BE-NEXT: rev32 v2.8b, v2.8b ; CHECK-BE-NEXT: uaddw v17.2d, v19.2d, v17.2s -; CHECK-BE-NEXT: ld1 { v19.2d }, [x12] -; CHECK-BE-NEXT: uaddw v5.2d, v21.2d, v5.2s -; CHECK-BE-NEXT: ld1 { v21.2d }, [x13] -; CHECK-BE-NEXT: uaddw v3.2d, v6.2d, v3.2s -; CHECK-BE-NEXT: ld1 { v6.2d }, [x17] -; CHECK-BE-NEXT: uaddw v4.2d, v16.2d, v4.2s +; CHECK-BE-NEXT: ld1 { v19.2d }, [x13] +; CHECK-BE-NEXT: uaddw v6.2d, v21.2d, v6.2s +; CHECK-BE-NEXT: uaddw v3.2d, v5.2d, v3.2s +; CHECK-BE-NEXT: ld1 { v5.2d }, [x17] ; CHECK-BE-NEXT: st1 { v7.2d }, [x15] -; CHECK-BE-NEXT: uaddw v7.2d, v19.2d, v18.2s -; CHECK-BE-NEXT: uaddw v16.2d, v21.2d, v20.2s -; CHECK-BE-NEXT: uaddw v2.2d, v6.2d, v2.2s -; CHECK-BE-NEXT: st1 { v17.2d }, [x14] -; CHECK-BE-NEXT: st1 { v5.2d }, [x11] +; CHECK-BE-NEXT: uaddw v4.2d, v16.2d, v4.2s +; CHECK-BE-NEXT: st1 { v6.2d }, [x11] +; CHECK-BE-NEXT: uaddw v6.2d, v22.2d, v18.2s ; CHECK-BE-NEXT: st1 { v3.2d }, [x10] +; CHECK-BE-NEXT: uaddw v3.2d, v19.2d, v20.2s +; CHECK-BE-NEXT: uaddw v2.2d, v5.2d, v2.2s +; CHECK-BE-NEXT: st1 { v17.2d }, [x14] ; CHECK-BE-NEXT: st1 { v4.2d }, [x16] -; CHECK-BE-NEXT: st1 { v7.2d }, [x12] -; CHECK-BE-NEXT: st1 { v16.2d }, [x13] +; CHECK-BE-NEXT: st1 { v6.2d }, [x12] +; CHECK-BE-NEXT: st1 { v3.2d }, [x13] ; CHECK-BE-NEXT: st1 { v2.2d }, [x17] ; CHECK-BE-NEXT: b.ne .LBB17_1 ; CHECK-BE-NEXT: // %bb.2: // %exit diff --git a/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-neon-instructions.s b/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-neon-instructions.s index 73eadc268bf2..f703392f3e9d 100644 --- a/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-neon-instructions.s +++ b/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-neon-instructions.s @@ -1070,14 +1070,14 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: [6]: HasSideEffects (U) # CHECK: [1] [2] [3] [4] [5] [6] Instructions: -# CHECK-NEXT: 1 4 0.50 abs d29, d24 -# CHECK-NEXT: 1 4 0.50 abs v0.16b, v0.16b -# CHECK-NEXT: 1 4 0.50 abs v0.2d, v0.2d -# CHECK-NEXT: 1 4 0.50 abs v0.2s, v0.2s -# CHECK-NEXT: 1 4 0.50 abs v0.4h, v0.4h -# CHECK-NEXT: 1 4 0.50 abs v0.4s, v0.4s -# CHECK-NEXT: 1 4 0.50 abs v0.8b, v0.8b -# CHECK-NEXT: 1 4 0.50 abs v0.8h, v0.8h +# CHECK-NEXT: 1 3 0.50 abs d29, d24 +# CHECK-NEXT: 1 3 0.50 abs v0.16b, v0.16b +# CHECK-NEXT: 1 3 0.50 abs v0.2d, v0.2d +# CHECK-NEXT: 1 3 0.50 abs v0.2s, v0.2s +# CHECK-NEXT: 1 3 0.50 abs v0.4h, v0.4h +# CHECK-NEXT: 1 3 0.50 abs v0.4s, v0.4s +# CHECK-NEXT: 1 3 0.50 abs v0.8b, v0.8b +# CHECK-NEXT: 1 3 0.50 abs v0.8h, v0.8h # CHECK-NEXT: 1 3 0.50 add d17, d31, d29 # CHECK-NEXT: 1 3 0.50 add v0.8b, v0.8b, v0.8b # CHECK-NEXT: 1 4 0.50 addhn v0.2s, v0.2d, v0.2d @@ -1086,8 +1086,8 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: 1 4 0.50 addhn2 v0.16b, v0.8h, v0.8h # CHECK-NEXT: 1 4 0.50 addhn2 v0.4s, v0.2d, v0.2d # CHECK-NEXT: 1 4 0.50 addhn2 v0.8h, v0.4s, v0.4s -# CHECK-NEXT: 1 4 0.50 addp v0.2d, v0.2d, v0.2d -# CHECK-NEXT: 1 4 0.50 addp v0.8b, v0.8b, v0.8b +# CHECK-NEXT: 1 3 0.50 addp v0.2d, v0.2d, v0.2d +# CHECK-NEXT: 1 3 0.50 addp v0.8b, v0.8b, v0.8b # CHECK-NEXT: 1 3 0.50 and v0.8b, v0.8b, v0.8b # CHECK-NEXT: 1 3 0.50 bic v0.4h, #15, lsl #8 # CHECK-NEXT: 1 3 0.50 bic v0.8b, v0.8b, v0.8b @@ -1441,13 +1441,13 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: 1 3 0.50 mvni v0.2s, #0 # CHECK-NEXT: 1 3 0.50 mvni v0.4s, #16, msl #16 # CHECK-NEXT: 1 3 0.50 neg d29, d24 -# CHECK-NEXT: 1 4 0.50 neg v0.16b, v0.16b -# CHECK-NEXT: 1 4 0.50 neg v0.2d, v0.2d +# CHECK-NEXT: 1 3 0.50 neg v0.16b, v0.16b +# CHECK-NEXT: 1 3 0.50 neg v0.2d, v0.2d # CHECK-NEXT: 1 3 0.50 neg v0.2s, v0.2s # CHECK-NEXT: 1 3 0.50 neg v0.4h, v0.4h -# CHECK-NEXT: 1 4 0.50 neg v0.4s, v0.4s +# CHECK-NEXT: 1 3 0.50 neg v0.4s, v0.4s # CHECK-NEXT: 1 3 0.50 neg v0.8b, v0.8b -# CHECK-NEXT: 1 4 0.50 neg v0.8h, v0.8h +# CHECK-NEXT: 1 3 0.50 neg v0.8h, v0.8h # CHECK-NEXT: 1 3 0.50 mvn v0.16b, v0.16b # CHECK-NEXT: 1 3 0.50 mvn v0.8b, v0.8b # CHECK-NEXT: 1 3 0.50 orn v0.16b, v0.16b, v0.16b @@ -1457,12 +1457,12 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: 1 4 0.50 pmul v0.8b, v0.8b, v0.8b # CHECK-NEXT: 1 4 0.50 pmull v0.8h, v0.8b, v0.8b # CHECK-NEXT: 1 4 0.50 pmull2 v0.8h, v0.16b, v0.16b -# CHECK-NEXT: 1 4 0.50 raddhn v0.2s, v0.2d, v0.2d -# CHECK-NEXT: 1 4 0.50 raddhn v0.4h, v0.4s, v0.4s -# CHECK-NEXT: 1 4 0.50 raddhn v0.8b, v0.8h, v0.8h -# CHECK-NEXT: 1 4 0.50 raddhn2 v0.16b, v0.8h, v0.8h -# CHECK-NEXT: 1 4 0.50 raddhn2 v0.4s, v0.2d, v0.2d -# CHECK-NEXT: 1 4 0.50 raddhn2 v0.8h, v0.4s, v0.4s +# CHECK-NEXT: 1 8 0.50 raddhn v0.2s, v0.2d, v0.2d +# CHECK-NEXT: 1 8 0.50 raddhn v0.4h, v0.4s, v0.4s +# CHECK-NEXT: 1 8 0.50 raddhn v0.8b, v0.8h, v0.8h +# CHECK-NEXT: 1 8 0.50 raddhn2 v0.16b, v0.8h, v0.8h +# CHECK-NEXT: 1 8 0.50 raddhn2 v0.4s, v0.2d, v0.2d +# CHECK-NEXT: 1 8 0.50 raddhn2 v0.8h, v0.4s, v0.4s # CHECK-NEXT: 1 4 0.50 rbit v0.16b, v0.16b # CHECK-NEXT: 1 4 0.50 rbit v0.8b, v0.8b # CHECK-NEXT: 1 4 0.50 rev16 v21.8b, v1.8b @@ -1483,19 +1483,19 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: 1 4 0.50 rshrn2 v0.16b, v0.8h, #3 # CHECK-NEXT: 1 4 0.50 rshrn2 v0.4s, v0.2d, #3 # CHECK-NEXT: 1 4 0.50 rshrn2 v0.8h, v0.4s, #3 -# CHECK-NEXT: 1 4 0.50 rsubhn v0.2s, v0.2d, v0.2d -# CHECK-NEXT: 1 4 0.50 rsubhn v0.4h, v0.4s, v0.4s -# CHECK-NEXT: 1 4 0.50 rsubhn v0.8b, v0.8h, v0.8h -# CHECK-NEXT: 1 4 0.50 rsubhn2 v0.16b, v0.8h, v0.8h -# CHECK-NEXT: 1 4 0.50 rsubhn2 v0.4s, v0.2d, v0.2d -# CHECK-NEXT: 1 4 0.50 rsubhn2 v0.8h, v0.4s, v0.4s -# CHECK-NEXT: 1 8 0.50 saba v0.16b, v0.16b, v0.16b -# CHECK-NEXT: 1 8 0.50 sabal v0.2d, v0.2s, v0.2s -# CHECK-NEXT: 1 8 0.50 sabal v0.4s, v0.4h, v0.4h -# CHECK-NEXT: 1 8 0.50 sabal v0.8h, v0.8b, v0.8b -# CHECK-NEXT: 1 8 0.50 sabal2 v0.2d, v0.4s, v0.4s -# CHECK-NEXT: 1 8 0.50 sabal2 v0.4s, v0.8h, v0.8h -# CHECK-NEXT: 1 8 0.50 sabal2 v0.8h, v0.16b, v0.16b +# CHECK-NEXT: 1 8 0.50 rsubhn v0.2s, v0.2d, v0.2d +# CHECK-NEXT: 1 8 0.50 rsubhn v0.4h, v0.4s, v0.4s +# CHECK-NEXT: 1 8 0.50 rsubhn v0.8b, v0.8h, v0.8h +# CHECK-NEXT: 1 8 0.50 rsubhn2 v0.16b, v0.8h, v0.8h +# CHECK-NEXT: 1 8 0.50 rsubhn2 v0.4s, v0.2d, v0.2d +# CHECK-NEXT: 1 8 0.50 rsubhn2 v0.8h, v0.4s, v0.4s +# CHECK-NEXT: 1 6 0.50 saba v0.16b, v0.16b, v0.16b +# CHECK-NEXT: 1 6 0.50 sabal v0.2d, v0.2s, v0.2s +# CHECK-NEXT: 1 6 0.50 sabal v0.4s, v0.4h, v0.4h +# CHECK-NEXT: 1 6 0.50 sabal v0.8h, v0.8b, v0.8b +# CHECK-NEXT: 1 6 0.50 sabal2 v0.2d, v0.4s, v0.4s +# CHECK-NEXT: 1 6 0.50 sabal2 v0.4s, v0.8h, v0.8h +# CHECK-NEXT: 1 6 0.50 sabal2 v0.8h, v0.16b, v0.16b # CHECK-NEXT: 1 3 0.50 sabd v0.4h, v0.4h, v0.4h # CHECK-NEXT: 1 3 0.50 sabdl v0.2d, v0.2s, v0.2s # CHECK-NEXT: 1 3 0.50 sabdl v0.4s, v0.4h, v0.4h @@ -1503,30 +1503,30 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: 1 3 0.50 sabdl2 v0.2d, v0.4s, v0.4s # CHECK-NEXT: 1 3 0.50 sabdl2 v0.4s, v0.8h, v0.8h # CHECK-NEXT: 1 3 0.50 sabdl2 v0.8h, v0.16b, v0.16b -# CHECK-NEXT: 1 8 1.00 sadalp v0.1d, v0.2s -# CHECK-NEXT: 1 8 1.00 sadalp v0.2d, v0.4s -# CHECK-NEXT: 1 8 1.00 sadalp v0.2s, v0.4h -# CHECK-NEXT: 1 8 1.00 sadalp v0.4h, v0.8b -# CHECK-NEXT: 1 8 1.00 sadalp v0.4s, v0.8h -# CHECK-NEXT: 1 8 1.00 sadalp v0.8h, v0.16b -# CHECK-NEXT: 1 4 0.50 saddl v0.2d, v0.2s, v0.2s -# CHECK-NEXT: 1 4 0.50 saddl v0.4s, v0.4h, v0.4h -# CHECK-NEXT: 1 4 0.50 saddl v0.8h, v0.8b, v0.8b -# CHECK-NEXT: 1 4 0.50 saddl2 v0.2d, v0.4s, v0.4s -# CHECK-NEXT: 1 4 0.50 saddl2 v0.4s, v0.8h, v0.8h -# CHECK-NEXT: 1 4 0.50 saddl2 v0.8h, v0.16b, v0.16b -# CHECK-NEXT: 1 4 0.50 saddlp v0.1d, v0.2s -# CHECK-NEXT: 1 4 0.50 saddlp v0.2d, v0.4s -# CHECK-NEXT: 1 4 0.50 saddlp v0.2s, v0.4h -# CHECK-NEXT: 1 4 0.50 saddlp v0.4h, v0.8b -# CHECK-NEXT: 1 4 0.50 saddlp v0.4s, v0.8h -# CHECK-NEXT: 1 4 0.50 saddlp v0.8h, v0.16b -# CHECK-NEXT: 1 4 0.50 saddw v0.2d, v0.2d, v0.2s -# CHECK-NEXT: 1 4 0.50 saddw v0.4s, v0.4s, v0.4h -# CHECK-NEXT: 1 4 0.50 saddw v0.8h, v0.8h, v0.8b -# CHECK-NEXT: 1 4 0.50 saddw2 v0.2d, v0.2d, v0.4s -# CHECK-NEXT: 1 4 0.50 saddw2 v0.4s, v0.4s, v0.8h -# CHECK-NEXT: 1 4 0.50 saddw2 v0.8h, v0.8h, v0.16b +# CHECK-NEXT: 1 7 1.00 sadalp v0.1d, v0.2s +# CHECK-NEXT: 1 7 1.00 sadalp v0.2d, v0.4s +# CHECK-NEXT: 1 7 1.00 sadalp v0.2s, v0.4h +# CHECK-NEXT: 1 7 1.00 sadalp v0.4h, v0.8b +# CHECK-NEXT: 1 7 1.00 sadalp v0.4s, v0.8h +# CHECK-NEXT: 1 7 1.00 sadalp v0.8h, v0.16b +# CHECK-NEXT: 1 3 0.50 saddl v0.2d, v0.2s, v0.2s +# CHECK-NEXT: 1 3 0.50 saddl v0.4s, v0.4h, v0.4h +# CHECK-NEXT: 1 3 0.50 saddl v0.8h, v0.8b, v0.8b +# CHECK-NEXT: 1 3 0.50 saddl2 v0.2d, v0.4s, v0.4s +# CHECK-NEXT: 1 3 0.50 saddl2 v0.4s, v0.8h, v0.8h +# CHECK-NEXT: 1 3 0.50 saddl2 v0.8h, v0.16b, v0.16b +# CHECK-NEXT: 1 3 0.50 saddlp v0.1d, v0.2s +# CHECK-NEXT: 1 3 0.50 saddlp v0.2d, v0.4s +# CHECK-NEXT: 1 3 0.50 saddlp v0.2s, v0.4h +# CHECK-NEXT: 1 3 0.50 saddlp v0.4h, v0.8b +# CHECK-NEXT: 1 3 0.50 saddlp v0.4s, v0.8h +# CHECK-NEXT: 1 3 0.50 saddlp v0.8h, v0.16b +# CHECK-NEXT: 1 3 0.50 saddw v0.2d, v0.2d, v0.2s +# CHECK-NEXT: 1 3 0.50 saddw v0.4s, v0.4s, v0.4h +# CHECK-NEXT: 1 3 0.50 saddw v0.8h, v0.8h, v0.8b +# CHECK-NEXT: 1 3 0.50 saddw2 v0.2d, v0.2d, v0.4s +# CHECK-NEXT: 1 3 0.50 saddw2 v0.4s, v0.4s, v0.8h +# CHECK-NEXT: 1 3 0.50 saddw2 v0.8h, v0.8h, v0.16b # CHECK-NEXT: 1 4 0.50 scvtf d21, d12 # CHECK-NEXT: 1 4 0.50 scvtf d21, d12, #64 # CHECK-NEXT: 1 4 0.50 scvtf s22, s13 @@ -1573,18 +1573,18 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: 1 4 0.50 sli v0.4s, v0.4s, #3 # CHECK-NEXT: 1 4 0.50 sli v0.8b, v0.8b, #3 # CHECK-NEXT: 1 4 0.50 sli v0.8h, v0.8h, #3 -# CHECK-NEXT: 1 4 0.50 smax v0.2s, v0.2s, v0.2s -# CHECK-NEXT: 1 4 0.50 smax v0.4h, v0.4h, v0.4h -# CHECK-NEXT: 1 4 0.50 smax v0.8b, v0.8b, v0.8b -# CHECK-NEXT: 1 4 0.50 smaxp v0.2s, v0.2s, v0.2s -# CHECK-NEXT: 1 4 0.50 smaxp v0.4h, v0.4h, v0.4h -# CHECK-NEXT: 1 4 0.50 smaxp v0.8b, v0.8b, v0.8b -# CHECK-NEXT: 1 4 0.50 smin v0.16b, v0.16b, v0.16b +# CHECK-NEXT: 1 3 0.50 smax v0.2s, v0.2s, v0.2s +# CHECK-NEXT: 1 3 0.50 smax v0.4h, v0.4h, v0.4h +# CHECK-NEXT: 1 3 0.50 smax v0.8b, v0.8b, v0.8b +# CHECK-NEXT: 1 3 0.50 smaxp v0.2s, v0.2s, v0.2s +# CHECK-NEXT: 1 3 0.50 smaxp v0.4h, v0.4h, v0.4h +# CHECK-NEXT: 1 3 0.50 smaxp v0.8b, v0.8b, v0.8b +# CHECK-NEXT: 1 3 0.50 smin v0.16b, v0.16b, v0.16b # CHECK-NEXT: 1 4 0.50 smin v0.4s, v0.4s, v0.4s -# CHECK-NEXT: 1 4 0.50 smin v0.8h, v0.8h, v0.8h -# CHECK-NEXT: 1 4 0.50 sminp v0.16b, v0.16b, v0.16b +# CHECK-NEXT: 1 3 0.50 smin v0.8h, v0.8h, v0.8h +# CHECK-NEXT: 1 3 0.50 sminp v0.16b, v0.16b, v0.16b # CHECK-NEXT: 1 4 0.50 sminp v0.4s, v0.4s, v0.4s -# CHECK-NEXT: 1 4 0.50 sminp v0.8h, v0.8h, v0.8h +# CHECK-NEXT: 1 3 0.50 sminp v0.8h, v0.8h, v0.8h # CHECK-NEXT: 1 4 0.50 smlal v0.2d, v0.2s, v0.2s # CHECK-NEXT: 1 4 0.50 smlal v0.4s, v0.4h, v0.4h # CHECK-NEXT: 1 4 0.50 smlal v0.8h, v0.8b, v0.8b @@ -1777,14 +1777,14 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: 1 3 0.50 srshr v0.4s, v0.4s, #3 # CHECK-NEXT: 1 3 0.50 srshr v0.8b, v0.8b, #3 # CHECK-NEXT: 1 3 0.50 srshr v0.8h, v0.8h, #3 -# CHECK-NEXT: 1 8 1.00 srsra d15, d11, #19 -# CHECK-NEXT: 1 8 1.00 srsra v0.16b, v0.16b, #3 -# CHECK-NEXT: 1 8 1.00 srsra v0.2d, v0.2d, #3 -# CHECK-NEXT: 1 8 1.00 srsra v0.2s, v0.2s, #3 -# CHECK-NEXT: 1 8 1.00 srsra v0.4h, v0.4h, #3 -# CHECK-NEXT: 1 8 1.00 srsra v0.4s, v0.4s, #3 -# CHECK-NEXT: 1 8 1.00 srsra v0.8b, v0.8b, #3 -# CHECK-NEXT: 1 8 1.00 srsra v0.8h, v0.8h, #3 +# CHECK-NEXT: 1 7 1.00 srsra d15, d11, #19 +# CHECK-NEXT: 1 7 1.00 srsra v0.16b, v0.16b, #3 +# CHECK-NEXT: 1 7 1.00 srsra v0.2d, v0.2d, #3 +# CHECK-NEXT: 1 7 1.00 srsra v0.2s, v0.2s, #3 +# CHECK-NEXT: 1 7 1.00 srsra v0.4h, v0.4h, #3 +# CHECK-NEXT: 1 7 1.00 srsra v0.4s, v0.4s, #3 +# CHECK-NEXT: 1 7 1.00 srsra v0.8b, v0.8b, #3 +# CHECK-NEXT: 1 7 1.00 srsra v0.8h, v0.8h, #3 # CHECK-NEXT: 1 3 0.50 sshl d31, d31, d31 # CHECK-NEXT: 1 3 0.50 sshl v0.2d, v0.2d, v0.2d # CHECK-NEXT: 1 3 0.50 sshl v0.2s, v0.2s, v0.2s @@ -1800,26 +1800,26 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: 1 3 0.50 sshr v0.4s, v0.4s, #3 # CHECK-NEXT: 1 3 0.50 sshr v0.8b, v0.8b, #3 # CHECK-NEXT: 1 3 0.50 sshr v0.8h, v0.8h, #3 -# CHECK-NEXT: 1 8 1.00 ssra d18, d12, #21 -# CHECK-NEXT: 1 8 1.00 ssra v0.16b, v0.16b, #3 -# CHECK-NEXT: 1 8 1.00 ssra v0.2d, v0.2d, #3 -# CHECK-NEXT: 1 8 1.00 ssra v0.2s, v0.2s, #3 -# CHECK-NEXT: 1 8 1.00 ssra v0.4h, v0.4h, #3 -# CHECK-NEXT: 1 8 1.00 ssra v0.4s, v0.4s, #3 -# CHECK-NEXT: 1 8 1.00 ssra v0.8b, v0.8b, #3 -# CHECK-NEXT: 1 8 1.00 ssra v0.8h, v0.8h, #3 -# CHECK-NEXT: 1 4 0.50 ssubl v0.2d, v0.2s, v0.2s -# CHECK-NEXT: 1 4 0.50 ssubl v0.4s, v0.4h, v0.4h -# CHECK-NEXT: 1 4 0.50 ssubl v0.8h, v0.8b, v0.8b -# CHECK-NEXT: 1 4 0.50 ssubl2 v0.2d, v0.4s, v0.4s -# CHECK-NEXT: 1 4 0.50 ssubl2 v0.4s, v0.8h, v0.8h -# CHECK-NEXT: 1 4 0.50 ssubl2 v0.8h, v0.16b, v0.16b -# CHECK-NEXT: 1 4 0.50 ssubw v0.2d, v0.2d, v0.2s -# CHECK-NEXT: 1 4 0.50 ssubw v0.4s, v0.4s, v0.4h -# CHECK-NEXT: 1 4 0.50 ssubw v0.8h, v0.8h, v0.8b -# CHECK-NEXT: 1 4 0.50 ssubw2 v0.2d, v0.2d, v0.4s -# CHECK-NEXT: 1 4 0.50 ssubw2 v0.4s, v0.4s, v0.8h -# CHECK-NEXT: 1 4 0.50 ssubw2 v0.8h, v0.8h, v0.16b +# CHECK-NEXT: 1 3 0.50 ssra d18, d12, #21 +# CHECK-NEXT: 1 3 0.50 ssra v0.16b, v0.16b, #3 +# CHECK-NEXT: 1 3 0.50 ssra v0.2d, v0.2d, #3 +# CHECK-NEXT: 1 3 0.50 ssra v0.2s, v0.2s, #3 +# CHECK-NEXT: 1 3 0.50 ssra v0.4h, v0.4h, #3 +# CHECK-NEXT: 1 3 0.50 ssra v0.4s, v0.4s, #3 +# CHECK-NEXT: 1 3 0.50 ssra v0.8b, v0.8b, #3 +# CHECK-NEXT: 1 3 0.50 ssra v0.8h, v0.8h, #3 +# CHECK-NEXT: 1 3 0.50 ssubl v0.2d, v0.2s, v0.2s +# CHECK-NEXT: 1 3 0.50 ssubl v0.4s, v0.4h, v0.4h +# CHECK-NEXT: 1 3 0.50 ssubl v0.8h, v0.8b, v0.8b +# CHECK-NEXT: 1 3 0.50 ssubl2 v0.2d, v0.4s, v0.4s +# CHECK-NEXT: 1 3 0.50 ssubl2 v0.4s, v0.8h, v0.8h +# CHECK-NEXT: 1 3 0.50 ssubl2 v0.8h, v0.16b, v0.16b +# CHECK-NEXT: 1 3 0.50 ssubw v0.2d, v0.2d, v0.2s +# CHECK-NEXT: 1 3 0.50 ssubw v0.4s, v0.4s, v0.4h +# CHECK-NEXT: 1 3 0.50 ssubw v0.8h, v0.8h, v0.8b +# CHECK-NEXT: 1 3 0.50 ssubw2 v0.2d, v0.2d, v0.4s +# CHECK-NEXT: 1 3 0.50 ssubw2 v0.4s, v0.4s, v0.8h +# CHECK-NEXT: 1 3 0.50 ssubw2 v0.8h, v0.8h, v0.16b # CHECK-NEXT: 1 4 1.00 * st1 { v0.16b }, [x0] # CHECK-NEXT: 2 5 2.00 * st1 { v0.2d, v1.2d, v2.2d }, [x0], #48 # CHECK-NEXT: 1 5 4.00 * st1 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0] @@ -1843,7 +1843,7 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: 1 5 2.00 * st4 { v0.b, v1.b, v2.b, v3.b }[9], [x0] # CHECK-NEXT: 2 5 2.00 * st4 { v0.b, v1.b, v2.b, v3.b }[9], [x0], x5 # CHECK-NEXT: 1 3 0.50 sub d15, d5, d16 -# CHECK-NEXT: 1 4 0.50 sub v0.2d, v0.2d, v0.2d +# CHECK-NEXT: 1 3 0.50 sub v0.2d, v0.2d, v0.2d # CHECK-NEXT: 1 4 0.50 suqadd b19, b14 # CHECK-NEXT: 1 4 0.50 suqadd d18, d22 # CHECK-NEXT: 1 4 0.50 suqadd h20, h15 @@ -1885,13 +1885,13 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: 1 4 0.50 trn2 v0.4s, v0.4s, v0.4s # CHECK-NEXT: 1 4 0.50 trn2 v0.8b, v0.8b, v0.8b # CHECK-NEXT: 1 4 0.50 trn2 v0.8h, v0.8h, v0.8h -# CHECK-NEXT: 1 8 0.50 uaba v0.8b, v0.8b, v0.8b -# CHECK-NEXT: 1 8 0.50 uabal v0.2d, v0.2s, v0.2s -# CHECK-NEXT: 1 8 0.50 uabal v0.4s, v0.4h, v0.4h -# CHECK-NEXT: 1 8 0.50 uabal v0.8h, v0.8b, v0.8b -# CHECK-NEXT: 1 8 0.50 uabal2 v0.2d, v0.4s, v0.4s -# CHECK-NEXT: 1 8 0.50 uabal2 v0.4s, v0.8h, v0.8h -# CHECK-NEXT: 1 8 0.50 uabal2 v0.8h, v0.16b, v0.16b +# CHECK-NEXT: 1 6 0.50 uaba v0.8b, v0.8b, v0.8b +# CHECK-NEXT: 1 6 0.50 uabal v0.2d, v0.2s, v0.2s +# CHECK-NEXT: 1 6 0.50 uabal v0.4s, v0.4h, v0.4h +# CHECK-NEXT: 1 6 0.50 uabal v0.8h, v0.8b, v0.8b +# CHECK-NEXT: 1 6 0.50 uabal2 v0.2d, v0.4s, v0.4s +# CHECK-NEXT: 1 6 0.50 uabal2 v0.4s, v0.8h, v0.8h +# CHECK-NEXT: 1 6 0.50 uabal2 v0.8h, v0.16b, v0.16b # CHECK-NEXT: 1 3 0.50 uabd v0.4h, v0.4h, v0.4h # CHECK-NEXT: 1 3 0.50 uabdl v0.2d, v0.2s, v0.2s # CHECK-NEXT: 1 3 0.50 uabdl v0.4s, v0.4h, v0.4h @@ -1899,30 +1899,30 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: 1 3 0.50 uabdl2 v0.2d, v0.4s, v0.4s # CHECK-NEXT: 1 3 0.50 uabdl2 v0.4s, v0.8h, v0.8h # CHECK-NEXT: 1 3 0.50 uabdl2 v0.8h, v0.16b, v0.16b -# CHECK-NEXT: 1 8 1.00 uadalp v0.1d, v0.2s -# CHECK-NEXT: 1 8 1.00 uadalp v0.2d, v0.4s -# CHECK-NEXT: 1 8 1.00 uadalp v0.2s, v0.4h -# CHECK-NEXT: 1 8 1.00 uadalp v0.4h, v0.8b -# CHECK-NEXT: 1 8 1.00 uadalp v0.4s, v0.8h -# CHECK-NEXT: 1 8 1.00 uadalp v0.8h, v0.16b -# CHECK-NEXT: 1 4 0.50 uaddl v0.2d, v0.2s, v0.2s -# CHECK-NEXT: 1 4 0.50 uaddl v0.4s, v0.4h, v0.4h -# CHECK-NEXT: 1 4 0.50 uaddl v0.8h, v0.8b, v0.8b -# CHECK-NEXT: 1 4 0.50 uaddl2 v0.2d, v0.4s, v0.4s -# CHECK-NEXT: 1 4 0.50 uaddl2 v0.4s, v0.8h, v0.8h -# CHECK-NEXT: 1 4 0.50 uaddl2 v0.8h, v0.16b, v0.16b -# CHECK-NEXT: 1 4 0.50 uaddlp v0.1d, v0.2s -# CHECK-NEXT: 1 4 0.50 uaddlp v0.2d, v0.4s -# CHECK-NEXT: 1 4 0.50 uaddlp v0.2s, v0.4h -# CHECK-NEXT: 1 4 0.50 uaddlp v0.4h, v0.8b -# CHECK-NEXT: 1 4 0.50 uaddlp v0.4s, v0.8h -# CHECK-NEXT: 1 4 0.50 uaddlp v0.8h, v0.16b -# CHECK-NEXT: 1 4 0.50 uaddw v0.2d, v0.2d, v0.2s -# CHECK-NEXT: 1 4 0.50 uaddw v0.4s, v0.4s, v0.4h -# CHECK-NEXT: 1 4 0.50 uaddw v0.8h, v0.8h, v0.8b -# CHECK-NEXT: 1 4 0.50 uaddw2 v0.2d, v0.2d, v0.4s -# CHECK-NEXT: 1 4 0.50 uaddw2 v0.4s, v0.4s, v0.8h -# CHECK-NEXT: 1 4 0.50 uaddw2 v0.8h, v0.8h, v0.16b +# CHECK-NEXT: 1 7 1.00 uadalp v0.1d, v0.2s +# CHECK-NEXT: 1 7 1.00 uadalp v0.2d, v0.4s +# CHECK-NEXT: 1 7 1.00 uadalp v0.2s, v0.4h +# CHECK-NEXT: 1 7 1.00 uadalp v0.4h, v0.8b +# CHECK-NEXT: 1 7 1.00 uadalp v0.4s, v0.8h +# CHECK-NEXT: 1 7 1.00 uadalp v0.8h, v0.16b +# CHECK-NEXT: 1 3 0.50 uaddl v0.2d, v0.2s, v0.2s +# CHECK-NEXT: 1 3 0.50 uaddl v0.4s, v0.4h, v0.4h +# CHECK-NEXT: 1 3 0.50 uaddl v0.8h, v0.8b, v0.8b +# CHECK-NEXT: 1 3 0.50 uaddl2 v0.2d, v0.4s, v0.4s +# CHECK-NEXT: 1 3 0.50 uaddl2 v0.4s, v0.8h, v0.8h +# CHECK-NEXT: 1 3 0.50 uaddl2 v0.8h, v0.16b, v0.16b +# CHECK-NEXT: 1 3 0.50 uaddlp v0.1d, v0.2s +# CHECK-NEXT: 1 3 0.50 uaddlp v0.2d, v0.4s +# CHECK-NEXT: 1 3 0.50 uaddlp v0.2s, v0.4h +# CHECK-NEXT: 1 3 0.50 uaddlp v0.4h, v0.8b +# CHECK-NEXT: 1 3 0.50 uaddlp v0.4s, v0.8h +# CHECK-NEXT: 1 3 0.50 uaddlp v0.8h, v0.16b +# CHECK-NEXT: 1 3 0.50 uaddw v0.2d, v0.2d, v0.2s +# CHECK-NEXT: 1 3 0.50 uaddw v0.4s, v0.4s, v0.4h +# CHECK-NEXT: 1 3 0.50 uaddw v0.8h, v0.8h, v0.8b +# CHECK-NEXT: 1 3 0.50 uaddw2 v0.2d, v0.2d, v0.4s +# CHECK-NEXT: 1 3 0.50 uaddw2 v0.4s, v0.4s, v0.8h +# CHECK-NEXT: 1 3 0.50 uaddw2 v0.8h, v0.8h, v0.16b # CHECK-NEXT: 1 4 0.50 ucvtf d21, d14 # CHECK-NEXT: 1 4 0.50 ucvtf d21, d14, #64 # CHECK-NEXT: 1 4 0.50 ucvtf s22, s13 @@ -1935,21 +1935,21 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: 1 4 0.50 ucvtf v0.4s, v0.4s # CHECK-NEXT: 1 4 0.50 ucvtf v0.4s, v0.4s, #3 # CHECK-NEXT: 1 4 0.50 ucvtf v0.8h, v0.8h -# CHECK-NEXT: 1 4 0.50 uhadd v0.16b, v0.16b, v0.16b -# CHECK-NEXT: 1 4 0.50 uhadd v0.8h, v0.8h, v0.8h -# CHECK-NEXT: 1 4 0.50 uhsub v0.4s, v0.4s, v0.4s -# CHECK-NEXT: 1 4 0.50 umax v0.16b, v0.16b, v0.16b +# CHECK-NEXT: 1 3 0.50 uhadd v0.16b, v0.16b, v0.16b +# CHECK-NEXT: 1 3 0.50 uhadd v0.8h, v0.8h, v0.8h +# CHECK-NEXT: 1 3 0.50 uhsub v0.4s, v0.4s, v0.4s +# CHECK-NEXT: 1 3 0.50 umax v0.16b, v0.16b, v0.16b # CHECK-NEXT: 1 4 0.50 umax v0.4s, v0.4s, v0.4s -# CHECK-NEXT: 1 4 0.50 umax v0.8h, v0.8h, v0.8h -# CHECK-NEXT: 1 4 0.50 umaxp v0.16b, v0.16b, v0.16b +# CHECK-NEXT: 1 3 0.50 umax v0.8h, v0.8h, v0.8h +# CHECK-NEXT: 1 3 0.50 umaxp v0.16b, v0.16b, v0.16b # CHECK-NEXT: 1 4 0.50 umaxp v0.4s, v0.4s, v0.4s -# CHECK-NEXT: 1 4 0.50 umaxp v0.8h, v0.8h, v0.8h -# CHECK-NEXT: 1 4 0.50 umin v0.2s, v0.2s, v0.2s -# CHECK-NEXT: 1 4 0.50 umin v0.4h, v0.4h, v0.4h -# CHECK-NEXT: 1 4 0.50 umin v0.8b, v0.8b, v0.8b -# CHECK-NEXT: 1 4 0.50 uminp v0.2s, v0.2s, v0.2s -# CHECK-NEXT: 1 4 0.50 uminp v0.4h, v0.4h, v0.4h -# CHECK-NEXT: 1 4 0.50 uminp v0.8b, v0.8b, v0.8b +# CHECK-NEXT: 1 3 0.50 umaxp v0.8h, v0.8h, v0.8h +# CHECK-NEXT: 1 3 0.50 umin v0.2s, v0.2s, v0.2s +# CHECK-NEXT: 1 3 0.50 umin v0.4h, v0.4h, v0.4h +# CHECK-NEXT: 1 3 0.50 umin v0.8b, v0.8b, v0.8b +# CHECK-NEXT: 1 3 0.50 uminp v0.2s, v0.2s, v0.2s +# CHECK-NEXT: 1 3 0.50 uminp v0.4h, v0.4h, v0.4h +# CHECK-NEXT: 1 3 0.50 uminp v0.8b, v0.8b, v0.8b # CHECK-NEXT: 1 4 0.50 umlal v0.2d, v0.2s, v0.2s # CHECK-NEXT: 1 4 0.50 umlal v0.4s, v0.4h, v0.4h # CHECK-NEXT: 1 4 0.50 umlal v0.8h, v0.8b, v0.8b @@ -2024,9 +2024,9 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: 1 4 0.50 uqxtn2 v0.8h, v0.4s # CHECK-NEXT: 1 4 0.50 urecpe v0.2s, v0.2s # CHECK-NEXT: 1 4 0.50 urecpe v0.4s, v0.4s -# CHECK-NEXT: 1 4 0.50 urhadd v0.16b, v0.16b, v0.16b -# CHECK-NEXT: 1 4 0.50 urhadd v0.4s, v0.4s, v0.4s -# CHECK-NEXT: 1 4 0.50 urhadd v0.8h, v0.8h, v0.8h +# CHECK-NEXT: 1 3 0.50 urhadd v0.16b, v0.16b, v0.16b +# CHECK-NEXT: 1 3 0.50 urhadd v0.4s, v0.4s, v0.4s +# CHECK-NEXT: 1 3 0.50 urhadd v0.8h, v0.8h, v0.8h # CHECK-NEXT: 1 3 0.50 urshl d8, d7, d4 # CHECK-NEXT: 1 3 0.50 urshl v0.16b, v0.16b, v0.16b # CHECK-NEXT: 1 3 0.50 urshl v0.2d, v0.2d, v0.2d @@ -2042,14 +2042,14 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: 1 3 0.50 urshr v0.8h, v0.8h, #3 # CHECK-NEXT: 1 12 9.00 ursqrte v0.2s, v0.2s # CHECK-NEXT: 1 12 9.00 ursqrte v0.4s, v0.4s -# CHECK-NEXT: 1 8 1.00 ursra d18, d10, #13 -# CHECK-NEXT: 1 8 1.00 ursra v0.16b, v0.16b, #3 -# CHECK-NEXT: 1 8 1.00 ursra v0.2d, v0.2d, #3 -# CHECK-NEXT: 1 8 1.00 ursra v0.2s, v0.2s, #3 -# CHECK-NEXT: 1 8 1.00 ursra v0.4h, v0.4h, #3 -# CHECK-NEXT: 1 8 1.00 ursra v0.4s, v0.4s, #3 -# CHECK-NEXT: 1 8 1.00 ursra v0.8b, v0.8b, #3 -# CHECK-NEXT: 1 8 1.00 ursra v0.8h, v0.8h, #3 +# CHECK-NEXT: 1 7 1.00 ursra d18, d10, #13 +# CHECK-NEXT: 1 7 1.00 ursra v0.16b, v0.16b, #3 +# CHECK-NEXT: 1 7 1.00 ursra v0.2d, v0.2d, #3 +# CHECK-NEXT: 1 7 1.00 ursra v0.2s, v0.2s, #3 +# CHECK-NEXT: 1 7 1.00 ursra v0.4h, v0.4h, #3 +# CHECK-NEXT: 1 7 1.00 ursra v0.4s, v0.4s, #3 +# CHECK-NEXT: 1 7 1.00 ursra v0.8b, v0.8b, #3 +# CHECK-NEXT: 1 7 1.00 ursra v0.8h, v0.8h, #3 # CHECK-NEXT: 1 3 0.50 ushl d0, d0, d0 # CHECK-NEXT: 1 3 0.50 ushl v0.16b, v0.16b, v0.16b # CHECK-NEXT: 1 3 0.50 ushl v0.4s, v0.4s, v0.4s @@ -2075,26 +2075,26 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: 1 4 0.50 usqadd v0.4s, v0.4s # CHECK-NEXT: 1 4 0.50 usqadd v0.8b, v0.8b # CHECK-NEXT: 1 4 0.50 usqadd v0.8h, v0.8h -# CHECK-NEXT: 1 8 1.00 usra d20, d13, #61 -# CHECK-NEXT: 1 8 1.00 usra v0.16b, v0.16b, #3 -# CHECK-NEXT: 1 8 1.00 usra v0.2d, v0.2d, #3 -# CHECK-NEXT: 1 8 1.00 usra v0.2s, v0.2s, #3 -# CHECK-NEXT: 1 8 1.00 usra v0.4h, v0.4h, #3 -# CHECK-NEXT: 1 8 1.00 usra v0.4s, v0.4s, #3 -# CHECK-NEXT: 1 8 1.00 usra v0.8b, v0.8b, #3 -# CHECK-NEXT: 1 8 1.00 usra v0.8h, v0.8h, #3 -# CHECK-NEXT: 1 4 0.50 usubl v0.2d, v0.2s, v0.2s -# CHECK-NEXT: 1 4 0.50 usubl v0.4s, v0.4h, v0.4h -# CHECK-NEXT: 1 4 0.50 usubl v0.8h, v0.8b, v0.8b -# CHECK-NEXT: 1 4 0.50 usubl2 v0.2d, v0.4s, v0.4s -# CHECK-NEXT: 1 4 0.50 usubl2 v0.4s, v0.8h, v0.8h -# CHECK-NEXT: 1 4 0.50 usubl2 v0.8h, v0.16b, v0.16b -# CHECK-NEXT: 1 4 0.50 usubw v0.2d, v0.2d, v0.2s -# CHECK-NEXT: 1 4 0.50 usubw v0.4s, v0.4s, v0.4h -# CHECK-NEXT: 1 4 0.50 usubw v0.8h, v0.8h, v0.8b -# CHECK-NEXT: 1 4 0.50 usubw2 v0.2d, v0.2d, v0.4s -# CHECK-NEXT: 1 4 0.50 usubw2 v0.4s, v0.4s, v0.8h -# CHECK-NEXT: 1 4 0.50 usubw2 v0.8h, v0.8h, v0.16b +# CHECK-NEXT: 1 3 0.50 usra d20, d13, #61 +# CHECK-NEXT: 1 3 0.50 usra v0.16b, v0.16b, #3 +# CHECK-NEXT: 1 3 0.50 usra v0.2d, v0.2d, #3 +# CHECK-NEXT: 1 3 0.50 usra v0.2s, v0.2s, #3 +# CHECK-NEXT: 1 3 0.50 usra v0.4h, v0.4h, #3 +# CHECK-NEXT: 1 3 0.50 usra v0.4s, v0.4s, #3 +# CHECK-NEXT: 1 3 0.50 usra v0.8b, v0.8b, #3 +# CHECK-NEXT: 1 3 0.50 usra v0.8h, v0.8h, #3 +# CHECK-NEXT: 1 3 0.50 usubl v0.2d, v0.2s, v0.2s +# CHECK-NEXT: 1 3 0.50 usubl v0.4s, v0.4h, v0.4h +# CHECK-NEXT: 1 3 0.50 usubl v0.8h, v0.8b, v0.8b +# CHECK-NEXT: 1 3 0.50 usubl2 v0.2d, v0.4s, v0.4s +# CHECK-NEXT: 1 3 0.50 usubl2 v0.4s, v0.8h, v0.8h +# CHECK-NEXT: 1 3 0.50 usubl2 v0.8h, v0.16b, v0.16b +# CHECK-NEXT: 1 3 0.50 usubw v0.2d, v0.2d, v0.2s +# CHECK-NEXT: 1 3 0.50 usubw v0.4s, v0.4s, v0.4h +# CHECK-NEXT: 1 3 0.50 usubw v0.8h, v0.8h, v0.8b +# CHECK-NEXT: 1 3 0.50 usubw2 v0.2d, v0.2d, v0.4s +# CHECK-NEXT: 1 3 0.50 usubw2 v0.4s, v0.4s, v0.8h +# CHECK-NEXT: 1 3 0.50 usubw2 v0.8h, v0.8h, v0.16b # CHECK-NEXT: 1 4 0.50 uzp1 v0.16b, v0.16b, v0.16b # CHECK-NEXT: 1 4 0.50 uzp1 v0.2d, v0.2d, v0.2d # CHECK-NEXT: 1 4 0.50 uzp1 v0.2s, v0.2s, v0.2s @@ -2148,7 +2148,7 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK: Resource pressure per iteration: # CHECK-NEXT: [0] [1.0] [1.1] [2] [3] [4] [5] [6] [7] [8] [9] [10.0] [10.1] [11] -# CHECK-NEXT: - - - - - 39.00 91.00 - - 509.00 509.00 3.00 3.00 197.00 +# CHECK-NEXT: - - - - - 39.00 91.00 - - 501.00 501.00 3.00 3.00 197.00 # CHECK: Resource pressure by instruction: # CHECK-NEXT: [0] [1.0] [1.1] [2] [3] [4] [5] [6] [7] [8] [9] [10.0] [10.1] [11] Instructions: @@ -2882,14 +2882,14 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - sshr v0.4s, v0.4s, #3 # CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - sshr v0.8b, v0.8b, #3 # CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - sshr v0.8h, v0.8h, #3 -# CHECK-NEXT: - - - - - - - - - 1.00 1.00 - - - ssra d18, d12, #21 -# CHECK-NEXT: - - - - - - - - - 1.00 1.00 - - - ssra v0.16b, v0.16b, #3 -# CHECK-NEXT: - - - - - - - - - 1.00 1.00 - - - ssra v0.2d, v0.2d, #3 -# CHECK-NEXT: - - - - - - - - - 1.00 1.00 - - - ssra v0.2s, v0.2s, #3 -# CHECK-NEXT: - - - - - - - - - 1.00 1.00 - - - ssra v0.4h, v0.4h, #3 -# CHECK-NEXT: - - - - - - - - - 1.00 1.00 - - - ssra v0.4s, v0.4s, #3 -# CHECK-NEXT: - - - - - - - - - 1.00 1.00 - - - ssra v0.8b, v0.8b, #3 -# CHECK-NEXT: - - - - - - - - - 1.00 1.00 - - - ssra v0.8h, v0.8h, #3 +# CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - ssra d18, d12, #21 +# CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - ssra v0.16b, v0.16b, #3 +# CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - ssra v0.2d, v0.2d, #3 +# CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - ssra v0.2s, v0.2s, #3 +# CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - ssra v0.4h, v0.4h, #3 +# CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - ssra v0.4s, v0.4s, #3 +# CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - ssra v0.8b, v0.8b, #3 +# CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - ssra v0.8h, v0.8h, #3 # CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - ssubl v0.2d, v0.2s, v0.2s # CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - ssubl v0.4s, v0.4h, v0.4h # CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - ssubl v0.8h, v0.8b, v0.8b @@ -3157,14 +3157,14 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - usqadd v0.4s, v0.4s # CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - usqadd v0.8b, v0.8b # CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - usqadd v0.8h, v0.8h -# CHECK-NEXT: - - - - - - - - - 1.00 1.00 - - - usra d20, d13, #61 -# CHECK-NEXT: - - - - - - - - - 1.00 1.00 - - - usra v0.16b, v0.16b, #3 -# CHECK-NEXT: - - - - - - - - - 1.00 1.00 - - - usra v0.2d, v0.2d, #3 -# CHECK-NEXT: - - - - - - - - - 1.00 1.00 - - - usra v0.2s, v0.2s, #3 -# CHECK-NEXT: - - - - - - - - - 1.00 1.00 - - - usra v0.4h, v0.4h, #3 -# CHECK-NEXT: - - - - - - - - - 1.00 1.00 - - - usra v0.4s, v0.4s, #3 -# CHECK-NEXT: - - - - - - - - - 1.00 1.00 - - - usra v0.8b, v0.8b, #3 -# CHECK-NEXT: - - - - - - - - - 1.00 1.00 - - - usra v0.8h, v0.8h, #3 +# CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - usra d20, d13, #61 +# CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - usra v0.16b, v0.16b, #3 +# CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - usra v0.2d, v0.2d, #3 +# CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - usra v0.2s, v0.2s, #3 +# CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - usra v0.4h, v0.4h, #3 +# CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - usra v0.4s, v0.4s, #3 +# CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - usra v0.8b, v0.8b, #3 +# CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - usra v0.8h, v0.8h, #3 # CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - usubl v0.2d, v0.2s, v0.2s # CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - usubl v0.4s, v0.4h, v0.4h # CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - usubl v0.8h, v0.8b, v0.8b diff --git a/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-sve-instructions.s b/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-sve-instructions.s index a8fb8b669838..d8051e7ecb4f 100644 --- a/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-sve-instructions.s +++ b/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-sve-instructions.s @@ -3476,12 +3476,12 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 add z31.s, p7/m, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 add z31.s, z31.s, #65280 # CHECK-NEXT: 1 3 0.50 add z31.s, z31.s, z31.s -# CHECK-NEXT: 1 4 0.50 addhnb z0.b, z1.h, z31.h -# CHECK-NEXT: 1 4 0.50 addhnb z0.h, z1.s, z31.s -# CHECK-NEXT: 1 4 0.50 addhnb z0.s, z1.d, z31.d -# CHECK-NEXT: 1 4 0.50 addhnt z0.b, z1.h, z31.h -# CHECK-NEXT: 1 4 0.50 addhnt z0.h, z1.s, z31.s -# CHECK-NEXT: 1 4 0.50 addhnt z0.s, z1.d, z31.d +# CHECK-NEXT: 1 8 0.50 addhnb z0.b, z1.h, z31.h +# CHECK-NEXT: 1 8 0.50 addhnb z0.h, z1.s, z31.s +# CHECK-NEXT: 1 8 0.50 addhnb z0.s, z1.d, z31.d +# CHECK-NEXT: 1 8 0.50 addhnt z0.b, z1.h, z31.h +# CHECK-NEXT: 1 8 0.50 addhnt z0.h, z1.s, z31.s +# CHECK-NEXT: 1 8 0.50 addhnt z0.s, z1.d, z31.d # CHECK-NEXT: 1 3 0.50 addp z0.b, p0/m, z0.b, z1.b # CHECK-NEXT: 1 3 0.50 addp z0.h, p0/m, z0.h, z1.h # CHECK-NEXT: 1 3 0.50 addp z29.s, p7/m, z29.s, z30.s @@ -3516,7 +3516,7 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 aesimc z31.b, z31.b # CHECK-NEXT: 1 3 0.50 aesmc z0.b, z0.b # CHECK-NEXT: 1 3 0.50 aesmc z31.b, z31.b -# CHECK-NEXT: 1 6 1.00 and p0.b, p0/z, p0.b, p1.b +# CHECK-NEXT: 1 2 1.00 and p0.b, p0/z, p0.b, p1.b # CHECK-NEXT: 1 3 0.50 and z0.d, z0.d, #0x6 # CHECK-NEXT: 1 3 0.50 and z0.d, z0.d, #0xfffffffffffffff9 # CHECK-NEXT: 1 3 0.50 and z0.d, z0.d, z0.d @@ -3531,7 +3531,7 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 and z31.s, p7/m, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 and z5.b, z5.b, #0x6 # CHECK-NEXT: 1 3 0.50 and z5.b, z5.b, #0xf9 -# CHECK-NEXT: 1 6 1.00 ands p0.b, p0/z, p0.b, p1.b +# CHECK-NEXT: 1 2 1.00 ands p0.b, p0/z, p0.b, p1.b # CHECK-NEXT: 1 4 1.00 andv b0, p7, z31.b # CHECK-NEXT: 1 4 1.00 andv d0, p7, z31.d # CHECK-NEXT: 1 4 1.00 andv h0, p7, z31.h @@ -3574,7 +3574,7 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 asrr z0.d, p0/m, z0.d, z0.d # CHECK-NEXT: 1 3 0.50 asrr z0.h, p0/m, z0.h, z0.h # CHECK-NEXT: 1 3 0.50 asrr z0.s, p0/m, z0.s, z0.s -# CHECK-NEXT: 1 3 0.50 bcax z29.d, z29.d, z30.d, z31.d +# CHECK-NEXT: 1 4 0.50 bcax z29.d, z29.d, z30.d, z31.d # CHECK-NEXT: 1 14 13.00 bdep z0.b, z1.b, z31.b # CHECK-NEXT: 1 70 69.00 bdep z0.d, z1.d, z31.d # CHECK-NEXT: 1 22 21.00 bdep z0.h, z1.h, z31.h @@ -3603,34 +3603,34 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 70 69.00 bgrp z0.d, z1.d, z31.d # CHECK-NEXT: 1 22 21.00 bgrp z0.h, z1.h, z31.h # CHECK-NEXT: 1 38 37.00 bgrp z0.s, z1.s, z31.s -# CHECK-NEXT: 1 6 1.00 bic p0.b, p0/z, p0.b, p0.b -# CHECK-NEXT: 1 6 1.00 bic p15.b, p15/z, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 bic p0.b, p0/z, p0.b, p0.b +# CHECK-NEXT: 1 2 1.00 bic p15.b, p15/z, p15.b, p15.b # CHECK-NEXT: 1 3 0.50 bic z0.d, z0.d, z0.d # CHECK-NEXT: 1 3 0.50 bic z23.d, z13.d, z8.d # CHECK-NEXT: 1 3 0.50 bic z31.b, p7/m, z31.b, z31.b # CHECK-NEXT: 1 3 0.50 bic z31.d, p7/m, z31.d, z31.d # CHECK-NEXT: 1 3 0.50 bic z31.h, p7/m, z31.h, z31.h # CHECK-NEXT: 1 3 0.50 bic z31.s, p7/m, z31.s, z31.s -# CHECK-NEXT: 1 6 1.00 bics p0.b, p0/z, p0.b, p0.b -# CHECK-NEXT: 1 6 1.00 bics p15.b, p15/z, p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 brka p0.b, p15/m, p15.b -# CHECK-NEXT: 1 6 1.00 brka p0.b, p15/z, p15.b -# CHECK-NEXT: 1 6 1.00 brkas p0.b, p15/z, p15.b -# CHECK-NEXT: 1 6 1.00 brkb p0.b, p15/m, p15.b -# CHECK-NEXT: 1 6 1.00 brkb p0.b, p15/z, p15.b -# CHECK-NEXT: 1 6 1.00 brkbs p0.b, p15/z, p15.b -# CHECK-NEXT: 1 6 1.00 brkn p0.b, p15/z, p1.b, p0.b -# CHECK-NEXT: 1 6 1.00 brkn p15.b, p15/z, p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 brkns p0.b, p15/z, p1.b, p0.b -# CHECK-NEXT: 1 6 1.00 brkns p15.b, p15/z, p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 brkpa p0.b, p15/z, p1.b, p2.b -# CHECK-NEXT: 1 6 1.00 brkpa p15.b, p15/z, p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 brkpas p0.b, p15/z, p1.b, p2.b -# CHECK-NEXT: 1 6 1.00 brkpas p15.b, p15/z, p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 brkpb p0.b, p15/z, p1.b, p2.b -# CHECK-NEXT: 1 6 1.00 brkpb p15.b, p15/z, p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 brkpbs p0.b, p15/z, p1.b, p2.b -# CHECK-NEXT: 1 6 1.00 brkpbs p15.b, p15/z, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 bics p0.b, p0/z, p0.b, p0.b +# CHECK-NEXT: 1 2 1.00 bics p15.b, p15/z, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 brka p0.b, p15/m, p15.b +# CHECK-NEXT: 1 2 1.00 brka p0.b, p15/z, p15.b +# CHECK-NEXT: 1 2 1.00 brkas p0.b, p15/z, p15.b +# CHECK-NEXT: 1 2 1.00 brkb p0.b, p15/m, p15.b +# CHECK-NEXT: 1 2 1.00 brkb p0.b, p15/z, p15.b +# CHECK-NEXT: 1 2 1.00 brkbs p0.b, p15/z, p15.b +# CHECK-NEXT: 1 2 1.00 brkn p0.b, p15/z, p1.b, p0.b +# CHECK-NEXT: 1 2 1.00 brkn p15.b, p15/z, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 brkns p0.b, p15/z, p1.b, p0.b +# CHECK-NEXT: 1 2 1.00 brkns p15.b, p15/z, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 brkpa p0.b, p15/z, p1.b, p2.b +# CHECK-NEXT: 1 2 1.00 brkpa p15.b, p15/z, p15.b, p15.b +# CHECK-NEXT: 1 4 1.00 brkpas p0.b, p15/z, p1.b, p2.b +# CHECK-NEXT: 1 4 1.00 brkpas p15.b, p15/z, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 brkpb p0.b, p15/z, p1.b, p2.b +# CHECK-NEXT: 1 2 1.00 brkpb p15.b, p15/z, p15.b, p15.b +# CHECK-NEXT: 1 4 1.00 brkpbs p0.b, p15/z, p1.b, p2.b +# CHECK-NEXT: 1 4 1.00 brkpbs p15.b, p15/z, p15.b, p15.b # CHECK-NEXT: 1 3 0.50 bsl z0.d, z0.d, z1.d, z2.d # CHECK-NEXT: 1 3 0.50 bsl1n z0.d, z0.d, z1.d, z2.d # CHECK-NEXT: 1 3 0.50 bsl2n z0.d, z0.d, z1.d, z2.d @@ -3704,163 +3704,163 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 4 0.50 cmla z31.h, z31.h, z31.h, #180 # CHECK-NEXT: 1 4 0.50 cmla z31.s, z30.s, z7.s[0], #180 # CHECK-NEXT: 1 4 0.50 cmla z31.s, z31.s, z31.s, #180 -# CHECK-NEXT: 1 3 0.50 cmpeq p0.b, p0/z, z0.b, #-16 -# CHECK-NEXT: 1 3 0.50 cmpeq p0.b, p0/z, z0.b, #15 -# CHECK-NEXT: 1 3 0.50 cmpeq p0.b, p0/z, z0.b, z0.b -# CHECK-NEXT: 1 3 0.50 cmpeq p0.b, p0/z, z0.b, z0.d -# CHECK-NEXT: 1 3 0.50 cmpeq p0.d, p0/z, z0.d, #-16 -# CHECK-NEXT: 1 3 0.50 cmpeq p0.d, p0/z, z0.d, #15 -# CHECK-NEXT: 1 3 0.50 cmpeq p0.d, p0/z, z0.d, z0.d -# CHECK-NEXT: 1 3 0.50 cmpeq p0.h, p0/z, z0.h, #-16 -# CHECK-NEXT: 1 3 0.50 cmpeq p0.h, p0/z, z0.h, #15 -# CHECK-NEXT: 1 3 0.50 cmpeq p0.h, p0/z, z0.h, z0.d -# CHECK-NEXT: 1 3 0.50 cmpeq p0.h, p0/z, z0.h, z0.h -# CHECK-NEXT: 1 3 0.50 cmpeq p0.s, p0/z, z0.s, #-16 -# CHECK-NEXT: 1 3 0.50 cmpeq p0.s, p0/z, z0.s, #15 -# CHECK-NEXT: 1 3 0.50 cmpeq p0.s, p0/z, z0.s, z0.d -# CHECK-NEXT: 1 3 0.50 cmpeq p0.s, p0/z, z0.s, z0.s -# CHECK-NEXT: 1 3 0.50 cmpge p0.b, p0/z, z0.b, #-16 -# CHECK-NEXT: 1 3 0.50 cmpge p0.b, p0/z, z0.b, #15 -# CHECK-NEXT: 1 3 0.50 cmpge p0.b, p0/z, z0.b, z0.b -# CHECK-NEXT: 1 3 0.50 cmpge p0.b, p0/z, z0.b, z0.d -# CHECK-NEXT: 1 3 0.50 cmpge p0.b, p0/z, z1.b, z0.b -# CHECK-NEXT: 1 3 0.50 cmpge p0.d, p0/z, z0.d, #-16 -# CHECK-NEXT: 1 3 0.50 cmpge p0.d, p0/z, z0.d, #15 -# CHECK-NEXT: 1 3 0.50 cmpge p0.d, p0/z, z0.d, z0.d -# CHECK-NEXT: 1 3 0.50 cmpge p0.d, p0/z, z1.d, z0.d -# CHECK-NEXT: 1 3 0.50 cmpge p0.h, p0/z, z0.h, #-16 -# CHECK-NEXT: 1 3 0.50 cmpge p0.h, p0/z, z0.h, #15 -# CHECK-NEXT: 1 3 0.50 cmpge p0.h, p0/z, z0.h, z0.d -# CHECK-NEXT: 1 3 0.50 cmpge p0.h, p0/z, z0.h, z0.h -# CHECK-NEXT: 1 3 0.50 cmpge p0.h, p0/z, z1.h, z0.h -# CHECK-NEXT: 1 3 0.50 cmpge p0.s, p0/z, z0.s, #-16 -# CHECK-NEXT: 1 3 0.50 cmpge p0.s, p0/z, z0.s, #15 -# CHECK-NEXT: 1 3 0.50 cmpge p0.s, p0/z, z0.s, z0.d -# CHECK-NEXT: 1 3 0.50 cmpge p0.s, p0/z, z0.s, z0.s -# CHECK-NEXT: 1 3 0.50 cmpge p0.s, p0/z, z1.s, z0.s -# CHECK-NEXT: 1 3 0.50 cmpgt p0.b, p0/z, z0.b, #-16 -# CHECK-NEXT: 1 3 0.50 cmpgt p0.b, p0/z, z0.b, #15 -# CHECK-NEXT: 1 3 0.50 cmpgt p0.b, p0/z, z0.b, z0.b -# CHECK-NEXT: 1 3 0.50 cmpgt p0.b, p0/z, z0.b, z0.d -# CHECK-NEXT: 1 3 0.50 cmpgt p0.b, p0/z, z1.b, z0.b -# CHECK-NEXT: 1 3 0.50 cmpgt p0.d, p0/z, z0.d, #-16 -# CHECK-NEXT: 1 3 0.50 cmpgt p0.d, p0/z, z0.d, #15 -# CHECK-NEXT: 1 3 0.50 cmpgt p0.d, p0/z, z0.d, z0.d -# CHECK-NEXT: 1 3 0.50 cmpgt p0.d, p0/z, z1.d, z0.d -# CHECK-NEXT: 1 3 0.50 cmpgt p0.h, p0/z, z0.h, #-16 -# CHECK-NEXT: 1 3 0.50 cmpgt p0.h, p0/z, z0.h, #15 -# CHECK-NEXT: 1 3 0.50 cmpgt p0.h, p0/z, z0.h, z0.d -# CHECK-NEXT: 1 3 0.50 cmpgt p0.h, p0/z, z0.h, z0.h -# CHECK-NEXT: 1 3 0.50 cmpgt p0.h, p0/z, z1.h, z0.h -# CHECK-NEXT: 1 3 0.50 cmpgt p0.s, p0/z, z0.s, #-16 -# CHECK-NEXT: 1 3 0.50 cmpgt p0.s, p0/z, z0.s, #15 -# CHECK-NEXT: 1 3 0.50 cmpgt p0.s, p0/z, z0.s, z0.d -# CHECK-NEXT: 1 3 0.50 cmpgt p0.s, p0/z, z0.s, z0.s -# CHECK-NEXT: 1 3 0.50 cmpgt p0.s, p0/z, z1.s, z0.s -# CHECK-NEXT: 1 3 0.50 cmphi p0.b, p0/z, z0.b, #0 -# CHECK-NEXT: 1 3 0.50 cmphi p0.b, p0/z, z0.b, #127 -# CHECK-NEXT: 1 3 0.50 cmphi p0.b, p0/z, z0.b, z0.b -# CHECK-NEXT: 1 3 0.50 cmphi p0.b, p0/z, z0.b, z0.d -# CHECK-NEXT: 1 3 0.50 cmphi p0.b, p0/z, z1.b, z0.b -# CHECK-NEXT: 1 3 0.50 cmphi p0.d, p0/z, z0.d, #0 -# CHECK-NEXT: 1 3 0.50 cmphi p0.d, p0/z, z0.d, #127 -# CHECK-NEXT: 1 3 0.50 cmphi p0.d, p0/z, z0.d, z0.d -# CHECK-NEXT: 1 3 0.50 cmphi p0.d, p0/z, z1.d, z0.d -# CHECK-NEXT: 1 3 0.50 cmphi p0.h, p0/z, z0.h, #0 -# CHECK-NEXT: 1 3 0.50 cmphi p0.h, p0/z, z0.h, #127 -# CHECK-NEXT: 1 3 0.50 cmphi p0.h, p0/z, z0.h, z0.d -# CHECK-NEXT: 1 3 0.50 cmphi p0.h, p0/z, z0.h, z0.h -# CHECK-NEXT: 1 3 0.50 cmphi p0.h, p0/z, z1.h, z0.h -# CHECK-NEXT: 1 3 0.50 cmphi p0.s, p0/z, z0.s, #0 -# CHECK-NEXT: 1 3 0.50 cmphi p0.s, p0/z, z0.s, #127 -# CHECK-NEXT: 1 3 0.50 cmphi p0.s, p0/z, z0.s, z0.d -# CHECK-NEXT: 1 3 0.50 cmphi p0.s, p0/z, z0.s, z0.s -# CHECK-NEXT: 1 3 0.50 cmphi p0.s, p0/z, z1.s, z0.s -# CHECK-NEXT: 1 3 0.50 cmphs p0.b, p0/z, z0.b, #0 -# CHECK-NEXT: 1 3 0.50 cmphs p0.b, p0/z, z0.b, #127 -# CHECK-NEXT: 1 3 0.50 cmphs p0.b, p0/z, z0.b, z0.b -# CHECK-NEXT: 1 3 0.50 cmphs p0.b, p0/z, z0.b, z0.d -# CHECK-NEXT: 1 3 0.50 cmphs p0.b, p0/z, z1.b, z0.b -# CHECK-NEXT: 1 3 0.50 cmphs p0.d, p0/z, z0.d, #0 -# CHECK-NEXT: 1 3 0.50 cmphs p0.d, p0/z, z0.d, #127 -# CHECK-NEXT: 1 3 0.50 cmphs p0.d, p0/z, z0.d, z0.d -# CHECK-NEXT: 1 3 0.50 cmphs p0.d, p0/z, z1.d, z0.d -# CHECK-NEXT: 1 3 0.50 cmphs p0.h, p0/z, z0.h, #0 -# CHECK-NEXT: 1 3 0.50 cmphs p0.h, p0/z, z0.h, #127 -# CHECK-NEXT: 1 3 0.50 cmphs p0.h, p0/z, z0.h, z0.d -# CHECK-NEXT: 1 3 0.50 cmphs p0.h, p0/z, z0.h, z0.h -# CHECK-NEXT: 1 3 0.50 cmphs p0.h, p0/z, z1.h, z0.h -# CHECK-NEXT: 1 3 0.50 cmphs p0.s, p0/z, z0.s, #0 -# CHECK-NEXT: 1 3 0.50 cmphs p0.s, p0/z, z0.s, #127 -# CHECK-NEXT: 1 3 0.50 cmphs p0.s, p0/z, z0.s, z0.d -# CHECK-NEXT: 1 3 0.50 cmphs p0.s, p0/z, z0.s, z0.s -# CHECK-NEXT: 1 3 0.50 cmphs p0.s, p0/z, z1.s, z0.s -# CHECK-NEXT: 1 3 0.50 cmple p0.b, p0/z, z0.b, #-16 -# CHECK-NEXT: 1 3 0.50 cmple p0.b, p0/z, z0.b, #15 -# CHECK-NEXT: 1 3 0.50 cmple p0.b, p0/z, z0.b, z0.d -# CHECK-NEXT: 1 3 0.50 cmple p0.d, p0/z, z0.d, #-16 -# CHECK-NEXT: 1 3 0.50 cmple p0.d, p0/z, z0.d, #15 -# CHECK-NEXT: 1 3 0.50 cmple p0.h, p0/z, z0.h, #-16 -# CHECK-NEXT: 1 3 0.50 cmple p0.h, p0/z, z0.h, #15 -# CHECK-NEXT: 1 3 0.50 cmple p0.h, p0/z, z0.h, z0.d -# CHECK-NEXT: 1 3 0.50 cmple p0.s, p0/z, z0.s, #-16 -# CHECK-NEXT: 1 3 0.50 cmple p0.s, p0/z, z0.s, #15 -# CHECK-NEXT: 1 3 0.50 cmple p0.s, p0/z, z0.s, z0.d -# CHECK-NEXT: 1 3 0.50 cmplo p0.b, p0/z, z0.b, #0 -# CHECK-NEXT: 1 3 0.50 cmplo p0.b, p0/z, z0.b, #127 -# CHECK-NEXT: 1 3 0.50 cmplo p0.b, p0/z, z0.b, z0.d -# CHECK-NEXT: 1 3 0.50 cmplo p0.d, p0/z, z0.d, #0 -# CHECK-NEXT: 1 3 0.50 cmplo p0.d, p0/z, z0.d, #127 -# CHECK-NEXT: 1 3 0.50 cmplo p0.h, p0/z, z0.h, #0 -# CHECK-NEXT: 1 3 0.50 cmplo p0.h, p0/z, z0.h, #127 -# CHECK-NEXT: 1 3 0.50 cmplo p0.h, p0/z, z0.h, z0.d -# CHECK-NEXT: 1 3 0.50 cmplo p0.s, p0/z, z0.s, #0 -# CHECK-NEXT: 1 3 0.50 cmplo p0.s, p0/z, z0.s, #127 -# CHECK-NEXT: 1 3 0.50 cmplo p0.s, p0/z, z0.s, z0.d -# CHECK-NEXT: 1 3 0.50 cmpls p0.b, p0/z, z0.b, #0 -# CHECK-NEXT: 1 3 0.50 cmpls p0.b, p0/z, z0.b, #127 -# CHECK-NEXT: 1 3 0.50 cmpls p0.b, p0/z, z0.b, z0.d -# CHECK-NEXT: 1 3 0.50 cmpls p0.d, p0/z, z0.d, #0 -# CHECK-NEXT: 1 3 0.50 cmpls p0.d, p0/z, z0.d, #127 -# CHECK-NEXT: 1 3 0.50 cmpls p0.h, p0/z, z0.h, #0 -# CHECK-NEXT: 1 3 0.50 cmpls p0.h, p0/z, z0.h, #127 -# CHECK-NEXT: 1 3 0.50 cmpls p0.h, p0/z, z0.h, z0.d -# CHECK-NEXT: 1 3 0.50 cmpls p0.s, p0/z, z0.s, #0 -# CHECK-NEXT: 1 3 0.50 cmpls p0.s, p0/z, z0.s, #127 -# CHECK-NEXT: 1 3 0.50 cmpls p0.s, p0/z, z0.s, z0.d -# CHECK-NEXT: 1 3 0.50 cmplt p0.b, p0/z, z0.b, #-16 -# CHECK-NEXT: 1 3 0.50 cmplt p0.b, p0/z, z0.b, #15 -# CHECK-NEXT: 1 3 0.50 cmplt p0.b, p0/z, z0.b, z0.d -# CHECK-NEXT: 1 3 0.50 cmplt p0.d, p0/z, z0.d, #-16 -# CHECK-NEXT: 1 3 0.50 cmplt p0.d, p0/z, z0.d, #15 -# CHECK-NEXT: 1 3 0.50 cmplt p0.h, p0/z, z0.h, #-16 -# CHECK-NEXT: 1 3 0.50 cmplt p0.h, p0/z, z0.h, #15 -# CHECK-NEXT: 1 3 0.50 cmplt p0.h, p0/z, z0.h, z0.d -# CHECK-NEXT: 1 3 0.50 cmplt p0.s, p0/z, z0.s, #-16 -# CHECK-NEXT: 1 3 0.50 cmplt p0.s, p0/z, z0.s, #15 -# CHECK-NEXT: 1 3 0.50 cmplt p0.s, p0/z, z0.s, z0.d -# CHECK-NEXT: 1 3 0.50 cmpne p0.b, p0/z, z0.b, #-16 -# CHECK-NEXT: 1 3 0.50 cmpne p0.b, p0/z, z0.b, #15 -# CHECK-NEXT: 1 3 0.50 cmpne p0.b, p0/z, z0.b, z0.b -# CHECK-NEXT: 1 3 0.50 cmpne p0.b, p0/z, z0.b, z0.d -# CHECK-NEXT: 1 3 0.50 cmpne p0.d, p0/z, z0.d, #-16 -# CHECK-NEXT: 1 3 0.50 cmpne p0.d, p0/z, z0.d, #15 -# CHECK-NEXT: 1 3 0.50 cmpne p0.d, p0/z, z0.d, z0.d -# CHECK-NEXT: 1 3 0.50 cmpne p0.h, p0/z, z0.h, #-16 -# CHECK-NEXT: 1 3 0.50 cmpne p0.h, p0/z, z0.h, #15 -# CHECK-NEXT: 1 3 0.50 cmpne p0.h, p0/z, z0.h, z0.d -# CHECK-NEXT: 1 3 0.50 cmpne p0.h, p0/z, z0.h, z0.h -# CHECK-NEXT: 1 3 0.50 cmpne p0.s, p0/z, z0.s, #-16 -# CHECK-NEXT: 1 3 0.50 cmpne p0.s, p0/z, z0.s, #15 -# CHECK-NEXT: 1 3 0.50 cmpne p0.s, p0/z, z0.s, z0.d -# CHECK-NEXT: 1 3 0.50 cmpne p0.s, p0/z, z0.s, z0.s +# CHECK-NEXT: 1 4 0.50 cmpeq p0.b, p0/z, z0.b, #-16 +# CHECK-NEXT: 1 4 0.50 cmpeq p0.b, p0/z, z0.b, #15 +# CHECK-NEXT: 1 4 0.50 cmpeq p0.b, p0/z, z0.b, z0.b +# CHECK-NEXT: 1 4 0.50 cmpeq p0.b, p0/z, z0.b, z0.d +# CHECK-NEXT: 1 4 0.50 cmpeq p0.d, p0/z, z0.d, #-16 +# CHECK-NEXT: 1 4 0.50 cmpeq p0.d, p0/z, z0.d, #15 +# CHECK-NEXT: 1 4 0.50 cmpeq p0.d, p0/z, z0.d, z0.d +# CHECK-NEXT: 1 4 0.50 cmpeq p0.h, p0/z, z0.h, #-16 +# CHECK-NEXT: 1 4 0.50 cmpeq p0.h, p0/z, z0.h, #15 +# CHECK-NEXT: 1 4 0.50 cmpeq p0.h, p0/z, z0.h, z0.d +# CHECK-NEXT: 1 4 0.50 cmpeq p0.h, p0/z, z0.h, z0.h +# CHECK-NEXT: 1 4 0.50 cmpeq p0.s, p0/z, z0.s, #-16 +# CHECK-NEXT: 1 4 0.50 cmpeq p0.s, p0/z, z0.s, #15 +# CHECK-NEXT: 1 4 0.50 cmpeq p0.s, p0/z, z0.s, z0.d +# CHECK-NEXT: 1 4 0.50 cmpeq p0.s, p0/z, z0.s, z0.s +# CHECK-NEXT: 1 4 0.50 cmpge p0.b, p0/z, z0.b, #-16 +# CHECK-NEXT: 1 4 0.50 cmpge p0.b, p0/z, z0.b, #15 +# CHECK-NEXT: 1 4 0.50 cmpge p0.b, p0/z, z0.b, z0.b +# CHECK-NEXT: 1 4 0.50 cmpge p0.b, p0/z, z0.b, z0.d +# CHECK-NEXT: 1 4 0.50 cmpge p0.b, p0/z, z1.b, z0.b +# CHECK-NEXT: 1 4 0.50 cmpge p0.d, p0/z, z0.d, #-16 +# CHECK-NEXT: 1 4 0.50 cmpge p0.d, p0/z, z0.d, #15 +# CHECK-NEXT: 1 4 0.50 cmpge p0.d, p0/z, z0.d, z0.d +# CHECK-NEXT: 1 4 0.50 cmpge p0.d, p0/z, z1.d, z0.d +# CHECK-NEXT: 1 4 0.50 cmpge p0.h, p0/z, z0.h, #-16 +# CHECK-NEXT: 1 4 0.50 cmpge p0.h, p0/z, z0.h, #15 +# CHECK-NEXT: 1 4 0.50 cmpge p0.h, p0/z, z0.h, z0.d +# CHECK-NEXT: 1 4 0.50 cmpge p0.h, p0/z, z0.h, z0.h +# CHECK-NEXT: 1 4 0.50 cmpge p0.h, p0/z, z1.h, z0.h +# CHECK-NEXT: 1 4 0.50 cmpge p0.s, p0/z, z0.s, #-16 +# CHECK-NEXT: 1 4 0.50 cmpge p0.s, p0/z, z0.s, #15 +# CHECK-NEXT: 1 4 0.50 cmpge p0.s, p0/z, z0.s, z0.d +# CHECK-NEXT: 1 4 0.50 cmpge p0.s, p0/z, z0.s, z0.s +# CHECK-NEXT: 1 4 0.50 cmpge p0.s, p0/z, z1.s, z0.s +# CHECK-NEXT: 1 4 0.50 cmpgt p0.b, p0/z, z0.b, #-16 +# CHECK-NEXT: 1 4 0.50 cmpgt p0.b, p0/z, z0.b, #15 +# CHECK-NEXT: 1 4 0.50 cmpgt p0.b, p0/z, z0.b, z0.b +# CHECK-NEXT: 1 4 0.50 cmpgt p0.b, p0/z, z0.b, z0.d +# CHECK-NEXT: 1 4 0.50 cmpgt p0.b, p0/z, z1.b, z0.b +# CHECK-NEXT: 1 4 0.50 cmpgt p0.d, p0/z, z0.d, #-16 +# CHECK-NEXT: 1 4 0.50 cmpgt p0.d, p0/z, z0.d, #15 +# CHECK-NEXT: 1 4 0.50 cmpgt p0.d, p0/z, z0.d, z0.d +# CHECK-NEXT: 1 4 0.50 cmpgt p0.d, p0/z, z1.d, z0.d +# CHECK-NEXT: 1 4 0.50 cmpgt p0.h, p0/z, z0.h, #-16 +# CHECK-NEXT: 1 4 0.50 cmpgt p0.h, p0/z, z0.h, #15 +# CHECK-NEXT: 1 4 0.50 cmpgt p0.h, p0/z, z0.h, z0.d +# CHECK-NEXT: 1 4 0.50 cmpgt p0.h, p0/z, z0.h, z0.h +# CHECK-NEXT: 1 4 0.50 cmpgt p0.h, p0/z, z1.h, z0.h +# CHECK-NEXT: 1 4 0.50 cmpgt p0.s, p0/z, z0.s, #-16 +# CHECK-NEXT: 1 4 0.50 cmpgt p0.s, p0/z, z0.s, #15 +# CHECK-NEXT: 1 4 0.50 cmpgt p0.s, p0/z, z0.s, z0.d +# CHECK-NEXT: 1 4 0.50 cmpgt p0.s, p0/z, z0.s, z0.s +# CHECK-NEXT: 1 4 0.50 cmpgt p0.s, p0/z, z1.s, z0.s +# CHECK-NEXT: 1 4 0.50 cmphi p0.b, p0/z, z0.b, #0 +# CHECK-NEXT: 1 4 0.50 cmphi p0.b, p0/z, z0.b, #127 +# CHECK-NEXT: 1 4 0.50 cmphi p0.b, p0/z, z0.b, z0.b +# CHECK-NEXT: 1 4 0.50 cmphi p0.b, p0/z, z0.b, z0.d +# CHECK-NEXT: 1 4 0.50 cmphi p0.b, p0/z, z1.b, z0.b +# CHECK-NEXT: 1 4 0.50 cmphi p0.d, p0/z, z0.d, #0 +# CHECK-NEXT: 1 4 0.50 cmphi p0.d, p0/z, z0.d, #127 +# CHECK-NEXT: 1 4 0.50 cmphi p0.d, p0/z, z0.d, z0.d +# CHECK-NEXT: 1 4 0.50 cmphi p0.d, p0/z, z1.d, z0.d +# CHECK-NEXT: 1 4 0.50 cmphi p0.h, p0/z, z0.h, #0 +# CHECK-NEXT: 1 4 0.50 cmphi p0.h, p0/z, z0.h, #127 +# CHECK-NEXT: 1 4 0.50 cmphi p0.h, p0/z, z0.h, z0.d +# CHECK-NEXT: 1 4 0.50 cmphi p0.h, p0/z, z0.h, z0.h +# CHECK-NEXT: 1 4 0.50 cmphi p0.h, p0/z, z1.h, z0.h +# CHECK-NEXT: 1 4 0.50 cmphi p0.s, p0/z, z0.s, #0 +# CHECK-NEXT: 1 4 0.50 cmphi p0.s, p0/z, z0.s, #127 +# CHECK-NEXT: 1 4 0.50 cmphi p0.s, p0/z, z0.s, z0.d +# CHECK-NEXT: 1 4 0.50 cmphi p0.s, p0/z, z0.s, z0.s +# CHECK-NEXT: 1 4 0.50 cmphi p0.s, p0/z, z1.s, z0.s +# CHECK-NEXT: 1 4 0.50 cmphs p0.b, p0/z, z0.b, #0 +# CHECK-NEXT: 1 4 0.50 cmphs p0.b, p0/z, z0.b, #127 +# CHECK-NEXT: 1 4 0.50 cmphs p0.b, p0/z, z0.b, z0.b +# CHECK-NEXT: 1 4 0.50 cmphs p0.b, p0/z, z0.b, z0.d +# CHECK-NEXT: 1 4 0.50 cmphs p0.b, p0/z, z1.b, z0.b +# CHECK-NEXT: 1 4 0.50 cmphs p0.d, p0/z, z0.d, #0 +# CHECK-NEXT: 1 4 0.50 cmphs p0.d, p0/z, z0.d, #127 +# CHECK-NEXT: 1 4 0.50 cmphs p0.d, p0/z, z0.d, z0.d +# CHECK-NEXT: 1 4 0.50 cmphs p0.d, p0/z, z1.d, z0.d +# CHECK-NEXT: 1 4 0.50 cmphs p0.h, p0/z, z0.h, #0 +# CHECK-NEXT: 1 4 0.50 cmphs p0.h, p0/z, z0.h, #127 +# CHECK-NEXT: 1 4 0.50 cmphs p0.h, p0/z, z0.h, z0.d +# CHECK-NEXT: 1 4 0.50 cmphs p0.h, p0/z, z0.h, z0.h +# CHECK-NEXT: 1 4 0.50 cmphs p0.h, p0/z, z1.h, z0.h +# CHECK-NEXT: 1 4 0.50 cmphs p0.s, p0/z, z0.s, #0 +# CHECK-NEXT: 1 4 0.50 cmphs p0.s, p0/z, z0.s, #127 +# CHECK-NEXT: 1 4 0.50 cmphs p0.s, p0/z, z0.s, z0.d +# CHECK-NEXT: 1 4 0.50 cmphs p0.s, p0/z, z0.s, z0.s +# CHECK-NEXT: 1 4 0.50 cmphs p0.s, p0/z, z1.s, z0.s +# CHECK-NEXT: 1 4 0.50 cmple p0.b, p0/z, z0.b, #-16 +# CHECK-NEXT: 1 4 0.50 cmple p0.b, p0/z, z0.b, #15 +# CHECK-NEXT: 1 4 0.50 cmple p0.b, p0/z, z0.b, z0.d +# CHECK-NEXT: 1 4 0.50 cmple p0.d, p0/z, z0.d, #-16 +# CHECK-NEXT: 1 4 0.50 cmple p0.d, p0/z, z0.d, #15 +# CHECK-NEXT: 1 4 0.50 cmple p0.h, p0/z, z0.h, #-16 +# CHECK-NEXT: 1 4 0.50 cmple p0.h, p0/z, z0.h, #15 +# CHECK-NEXT: 1 4 0.50 cmple p0.h, p0/z, z0.h, z0.d +# CHECK-NEXT: 1 4 0.50 cmple p0.s, p0/z, z0.s, #-16 +# CHECK-NEXT: 1 4 0.50 cmple p0.s, p0/z, z0.s, #15 +# CHECK-NEXT: 1 4 0.50 cmple p0.s, p0/z, z0.s, z0.d +# CHECK-NEXT: 1 4 0.50 cmplo p0.b, p0/z, z0.b, #0 +# CHECK-NEXT: 1 4 0.50 cmplo p0.b, p0/z, z0.b, #127 +# CHECK-NEXT: 1 4 0.50 cmplo p0.b, p0/z, z0.b, z0.d +# CHECK-NEXT: 1 4 0.50 cmplo p0.d, p0/z, z0.d, #0 +# CHECK-NEXT: 1 4 0.50 cmplo p0.d, p0/z, z0.d, #127 +# CHECK-NEXT: 1 4 0.50 cmplo p0.h, p0/z, z0.h, #0 +# CHECK-NEXT: 1 4 0.50 cmplo p0.h, p0/z, z0.h, #127 +# CHECK-NEXT: 1 4 0.50 cmplo p0.h, p0/z, z0.h, z0.d +# CHECK-NEXT: 1 4 0.50 cmplo p0.s, p0/z, z0.s, #0 +# CHECK-NEXT: 1 4 0.50 cmplo p0.s, p0/z, z0.s, #127 +# CHECK-NEXT: 1 4 0.50 cmplo p0.s, p0/z, z0.s, z0.d +# CHECK-NEXT: 1 4 0.50 cmpls p0.b, p0/z, z0.b, #0 +# CHECK-NEXT: 1 4 0.50 cmpls p0.b, p0/z, z0.b, #127 +# CHECK-NEXT: 1 4 0.50 cmpls p0.b, p0/z, z0.b, z0.d +# CHECK-NEXT: 1 4 0.50 cmpls p0.d, p0/z, z0.d, #0 +# CHECK-NEXT: 1 4 0.50 cmpls p0.d, p0/z, z0.d, #127 +# CHECK-NEXT: 1 4 0.50 cmpls p0.h, p0/z, z0.h, #0 +# CHECK-NEXT: 1 4 0.50 cmpls p0.h, p0/z, z0.h, #127 +# CHECK-NEXT: 1 4 0.50 cmpls p0.h, p0/z, z0.h, z0.d +# CHECK-NEXT: 1 4 0.50 cmpls p0.s, p0/z, z0.s, #0 +# CHECK-NEXT: 1 4 0.50 cmpls p0.s, p0/z, z0.s, #127 +# CHECK-NEXT: 1 4 0.50 cmpls p0.s, p0/z, z0.s, z0.d +# CHECK-NEXT: 1 4 0.50 cmplt p0.b, p0/z, z0.b, #-16 +# CHECK-NEXT: 1 4 0.50 cmplt p0.b, p0/z, z0.b, #15 +# CHECK-NEXT: 1 4 0.50 cmplt p0.b, p0/z, z0.b, z0.d +# CHECK-NEXT: 1 4 0.50 cmplt p0.d, p0/z, z0.d, #-16 +# CHECK-NEXT: 1 4 0.50 cmplt p0.d, p0/z, z0.d, #15 +# CHECK-NEXT: 1 4 0.50 cmplt p0.h, p0/z, z0.h, #-16 +# CHECK-NEXT: 1 4 0.50 cmplt p0.h, p0/z, z0.h, #15 +# CHECK-NEXT: 1 4 0.50 cmplt p0.h, p0/z, z0.h, z0.d +# CHECK-NEXT: 1 4 0.50 cmplt p0.s, p0/z, z0.s, #-16 +# CHECK-NEXT: 1 4 0.50 cmplt p0.s, p0/z, z0.s, #15 +# CHECK-NEXT: 1 4 0.50 cmplt p0.s, p0/z, z0.s, z0.d +# CHECK-NEXT: 1 4 0.50 cmpne p0.b, p0/z, z0.b, #-16 +# CHECK-NEXT: 1 4 0.50 cmpne p0.b, p0/z, z0.b, #15 +# CHECK-NEXT: 1 4 0.50 cmpne p0.b, p0/z, z0.b, z0.b +# CHECK-NEXT: 1 4 0.50 cmpne p0.b, p0/z, z0.b, z0.d +# CHECK-NEXT: 1 4 0.50 cmpne p0.d, p0/z, z0.d, #-16 +# CHECK-NEXT: 1 4 0.50 cmpne p0.d, p0/z, z0.d, #15 +# CHECK-NEXT: 1 4 0.50 cmpne p0.d, p0/z, z0.d, z0.d +# CHECK-NEXT: 1 4 0.50 cmpne p0.h, p0/z, z0.h, #-16 +# CHECK-NEXT: 1 4 0.50 cmpne p0.h, p0/z, z0.h, #15 +# CHECK-NEXT: 1 4 0.50 cmpne p0.h, p0/z, z0.h, z0.d +# CHECK-NEXT: 1 4 0.50 cmpne p0.h, p0/z, z0.h, z0.h +# CHECK-NEXT: 1 4 0.50 cmpne p0.s, p0/z, z0.s, #-16 +# CHECK-NEXT: 1 4 0.50 cmpne p0.s, p0/z, z0.s, #15 +# CHECK-NEXT: 1 4 0.50 cmpne p0.s, p0/z, z0.s, z0.d +# CHECK-NEXT: 1 4 0.50 cmpne p0.s, p0/z, z0.s, z0.s # CHECK-NEXT: 1 3 0.50 cnot z31.b, p7/m, z31.b # CHECK-NEXT: 1 3 0.50 cnot z31.d, p7/m, z31.d # CHECK-NEXT: 1 3 0.50 cnot z31.h, p7/m, z31.h # CHECK-NEXT: 1 3 0.50 cnot z31.s, p7/m, z31.s -# CHECK-NEXT: 1 4 0.50 cnt z31.b, p7/m, z31.b +# CHECK-NEXT: 1 3 0.50 cnt z31.b, p7/m, z31.b # CHECK-NEXT: 1 12 0.50 cnt z31.d, p7/m, z31.d -# CHECK-NEXT: 1 4 0.50 cnt z31.h, p7/m, z31.h +# CHECK-NEXT: 1 3 0.50 cnt z31.h, p7/m, z31.h # CHECK-NEXT: 1 8 0.50 cnt z31.s, p7/m, z31.s # CHECK-NEXT: 1 1 0.33 cntb x0 # CHECK-NEXT: 1 1 0.33 cntb x0, #28 @@ -3874,10 +3874,10 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 1 0.33 cnth x0, #28 # CHECK-NEXT: 1 1 0.33 cnth x0, all, mul #16 # CHECK-NEXT: 1 1 0.33 cnth x0, pow2 -# CHECK-NEXT: 1 6 1.00 cntp x0, p15, p0.b -# CHECK-NEXT: 1 6 1.00 cntp x0, p15, p0.d -# CHECK-NEXT: 1 6 1.00 cntp x0, p15, p0.h -# CHECK-NEXT: 1 6 1.00 cntp x0, p15, p0.s +# CHECK-NEXT: 1 4 1.00 cntp x0, p15, p0.b +# CHECK-NEXT: 1 4 1.00 cntp x0, p15, p0.d +# CHECK-NEXT: 1 4 1.00 cntp x0, p15, p0.h +# CHECK-NEXT: 1 4 1.00 cntp x0, p15, p0.s # CHECK-NEXT: 1 1 0.33 cntw x0 # CHECK-NEXT: 1 1 0.33 cntw x0, #28 # CHECK-NEXT: 1 1 0.33 cntw x0, all, mul #16 @@ -3892,42 +3892,42 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 1 0.33 ctermne wzr, w30 # CHECK-NEXT: 1 1 0.33 ctermne x30, xzr # CHECK-NEXT: 1 1 0.33 ctermne xzr, x30 -# CHECK-NEXT: 1 1 0.33 decb x0 -# CHECK-NEXT: 1 1 0.33 decb x0, #14 -# CHECK-NEXT: 1 1 0.33 decb x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 decb x0, pow2 -# CHECK-NEXT: 1 1 0.33 decb x0, vl1 -# CHECK-NEXT: 1 1 0.33 decd x0 -# CHECK-NEXT: 1 1 0.33 decd x0, #14 -# CHECK-NEXT: 1 1 0.33 decd x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 decd x0, pow2 -# CHECK-NEXT: 1 1 0.33 decd x0, vl1 -# CHECK-NEXT: 1 1 0.33 dech x0 -# CHECK-NEXT: 1 1 0.33 dech x0, #14 -# CHECK-NEXT: 1 1 0.33 dech x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 dech x0, pow2 -# CHECK-NEXT: 1 1 0.33 dech x0, vl1 -# CHECK-NEXT: 1 6 1.00 decp x0, p0.b -# CHECK-NEXT: 1 6 1.00 decp x0, p0.d -# CHECK-NEXT: 1 6 1.00 decp x0, p0.h -# CHECK-NEXT: 1 6 1.00 decp x0, p0.s -# CHECK-NEXT: 1 6 1.00 decp xzr, p15.b -# CHECK-NEXT: 1 6 1.00 decp xzr, p15.d -# CHECK-NEXT: 1 6 1.00 decp xzr, p15.h -# CHECK-NEXT: 1 6 1.00 decp xzr, p15.s +# CHECK-NEXT: 1 3 0.33 decb x0 +# CHECK-NEXT: 1 3 0.33 decb x0, #14 +# CHECK-NEXT: 1 3 0.33 decb x0, all, mul #16 +# CHECK-NEXT: 1 3 0.33 decb x0, pow2 +# CHECK-NEXT: 1 3 0.33 decb x0, vl1 +# CHECK-NEXT: 1 3 0.33 decd x0 +# CHECK-NEXT: 1 3 0.33 decd x0, #14 +# CHECK-NEXT: 1 3 0.33 decd x0, all, mul #16 +# CHECK-NEXT: 1 3 0.33 decd x0, pow2 +# CHECK-NEXT: 1 3 0.33 decd x0, vl1 +# CHECK-NEXT: 1 3 0.33 dech x0 +# CHECK-NEXT: 1 3 0.33 dech x0, #14 +# CHECK-NEXT: 1 3 0.33 dech x0, all, mul #16 +# CHECK-NEXT: 1 3 0.33 dech x0, pow2 +# CHECK-NEXT: 1 3 0.33 dech x0, vl1 +# CHECK-NEXT: 1 4 1.00 decp x0, p0.b +# CHECK-NEXT: 1 4 1.00 decp x0, p0.d +# CHECK-NEXT: 1 4 1.00 decp x0, p0.h +# CHECK-NEXT: 1 4 1.00 decp x0, p0.s +# CHECK-NEXT: 1 4 1.00 decp xzr, p15.b +# CHECK-NEXT: 1 4 1.00 decp xzr, p15.d +# CHECK-NEXT: 1 4 1.00 decp xzr, p15.h +# CHECK-NEXT: 1 4 1.00 decp xzr, p15.s # CHECK-NEXT: 1 4 0.50 decp z31.d, p15.d # CHECK-NEXT: 1 4 0.50 decp z31.h, p15.h # CHECK-NEXT: 1 4 0.50 decp z31.s, p15.s -# CHECK-NEXT: 1 1 0.33 decw x0 -# CHECK-NEXT: 1 1 0.33 decw x0, #14 -# CHECK-NEXT: 1 1 0.33 decw x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 decw x0, pow2 -# CHECK-NEXT: 1 1 0.33 decw x0, vl1 +# CHECK-NEXT: 1 3 0.33 decw x0 +# CHECK-NEXT: 1 3 0.33 decw x0, #14 +# CHECK-NEXT: 1 3 0.33 decw x0, all, mul #16 +# CHECK-NEXT: 1 3 0.33 decw x0, pow2 +# CHECK-NEXT: 1 3 0.33 decw x0, vl1 # CHECK-NEXT: 1 4 0.50 dupm z0.d, #0xfffffffffffffff9 # CHECK-NEXT: 1 4 0.50 dupm z0.s, #0xfffffff9 # CHECK-NEXT: 1 4 0.50 dupm z23.h, #0xfff9 # CHECK-NEXT: 1 4 0.50 dupm z5.b, #0xf9 -# CHECK-NEXT: 1 6 1.00 eor p0.b, p0/z, p0.b, p1.b +# CHECK-NEXT: 1 2 1.00 eor p0.b, p0/z, p0.b, p1.b # CHECK-NEXT: 1 3 0.50 eor z0.d, z0.d, #0x6 # CHECK-NEXT: 1 3 0.50 eor z0.d, z0.d, #0xfffffffffffffff9 # CHECK-NEXT: 1 3 0.50 eor z0.d, z0.d, z0.d @@ -3942,12 +3942,12 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 eor z31.s, p7/m, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 eor z5.b, z5.b, #0x6 # CHECK-NEXT: 1 3 0.50 eor z5.b, z5.b, #0xf9 -# CHECK-NEXT: 1 3 0.50 eor3 z29.d, z29.d, z30.d, z31.d +# CHECK-NEXT: 1 4 0.50 eor3 z29.d, z29.d, z30.d, z31.d # CHECK-NEXT: 1 4 0.50 eorbt z0.b, z1.b, z31.b # CHECK-NEXT: 1 4 0.50 eorbt z0.d, z1.d, z31.d # CHECK-NEXT: 1 4 0.50 eorbt z0.h, z1.h, z31.h # CHECK-NEXT: 1 4 0.50 eorbt z0.s, z1.s, z31.s -# CHECK-NEXT: 1 6 1.00 eors p0.b, p0/z, p0.b, p1.b +# CHECK-NEXT: 1 2 1.00 eors p0.b, p0/z, p0.b, p1.b # CHECK-NEXT: 1 4 0.50 eortb z0.b, z1.b, z31.b # CHECK-NEXT: 1 4 0.50 eortb z0.d, z1.d, z31.d # CHECK-NEXT: 1 4 0.50 eortb z0.h, z1.h, z31.h @@ -4303,49 +4303,49 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 4 0.50 ftsmul z0.d, z1.d, z31.d # CHECK-NEXT: 1 4 0.50 ftsmul z0.h, z1.h, z31.h # CHECK-NEXT: 1 4 0.50 ftsmul z0.s, z1.s, z31.s -# CHECK-NEXT: 1 4 0.50 ftssel z0.d, z1.d, z31.d -# CHECK-NEXT: 1 4 0.50 ftssel z0.h, z1.h, z31.h -# CHECK-NEXT: 1 4 0.50 ftssel z0.s, z1.s, z31.s +# CHECK-NEXT: 1 3 0.50 ftssel z0.d, z1.d, z31.d +# CHECK-NEXT: 1 3 0.50 ftssel z0.h, z1.h, z31.h +# CHECK-NEXT: 1 3 0.50 ftssel z0.s, z1.s, z31.s # CHECK-NEXT: 1 8 2.00 histcnt z0.s, p0/z, z1.s, z2.s # CHECK-NEXT: 1 8 2.00 histcnt z29.d, p7/z, z30.d, z31.d # CHECK-NEXT: 1 8 2.00 histseg z0.b, z1.b, z31.b -# CHECK-NEXT: 1 1 0.33 incb x0 -# CHECK-NEXT: 1 1 0.33 incb x0, #14 -# CHECK-NEXT: 1 1 0.33 incb x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 incb x0, pow2 -# CHECK-NEXT: 1 1 0.33 incb x0, vl1 -# CHECK-NEXT: 1 1 0.33 incd x0 -# CHECK-NEXT: 1 1 0.33 incd x0, #14 -# CHECK-NEXT: 1 1 0.33 incd x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 incd x0, pow2 -# CHECK-NEXT: 1 1 0.33 incd x0, vl1 -# CHECK-NEXT: 1 4 0.50 incd z0.d -# CHECK-NEXT: 1 4 0.50 incd z0.d, all, mul #16 -# CHECK-NEXT: 1 1 0.33 inch x0 -# CHECK-NEXT: 1 1 0.33 inch x0, #14 -# CHECK-NEXT: 1 1 0.33 inch x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 inch x0, pow2 -# CHECK-NEXT: 1 1 0.33 inch x0, vl1 -# CHECK-NEXT: 1 4 0.50 inch z0.h -# CHECK-NEXT: 1 4 0.50 inch z0.h, all, mul #16 -# CHECK-NEXT: 1 6 1.00 incp x0, p0.b -# CHECK-NEXT: 1 6 1.00 incp x0, p0.d -# CHECK-NEXT: 1 6 1.00 incp x0, p0.h -# CHECK-NEXT: 1 6 1.00 incp x0, p0.s -# CHECK-NEXT: 1 6 1.00 incp xzr, p15.b -# CHECK-NEXT: 1 6 1.00 incp xzr, p15.d -# CHECK-NEXT: 1 6 1.00 incp xzr, p15.h -# CHECK-NEXT: 1 6 1.00 incp xzr, p15.s +# CHECK-NEXT: 1 3 0.33 incb x0 +# CHECK-NEXT: 1 3 0.33 incb x0, #14 +# CHECK-NEXT: 1 3 0.33 incb x0, all, mul #16 +# CHECK-NEXT: 1 3 0.33 incb x0, pow2 +# CHECK-NEXT: 1 3 0.33 incb x0, vl1 +# CHECK-NEXT: 1 3 0.33 incd x0 +# CHECK-NEXT: 1 3 0.33 incd x0, #14 +# CHECK-NEXT: 1 3 0.33 incd x0, all, mul #16 +# CHECK-NEXT: 1 3 0.33 incd x0, pow2 +# CHECK-NEXT: 1 3 0.33 incd x0, vl1 +# CHECK-NEXT: 1 3 0.50 incd z0.d +# CHECK-NEXT: 1 3 0.50 incd z0.d, all, mul #16 +# CHECK-NEXT: 1 3 0.33 inch x0 +# CHECK-NEXT: 1 3 0.33 inch x0, #14 +# CHECK-NEXT: 1 3 0.33 inch x0, all, mul #16 +# CHECK-NEXT: 1 3 0.33 inch x0, pow2 +# CHECK-NEXT: 1 3 0.33 inch x0, vl1 +# CHECK-NEXT: 1 3 0.50 inch z0.h +# CHECK-NEXT: 1 3 0.50 inch z0.h, all, mul #16 +# CHECK-NEXT: 1 4 1.00 incp x0, p0.b +# CHECK-NEXT: 1 4 1.00 incp x0, p0.d +# CHECK-NEXT: 1 4 1.00 incp x0, p0.h +# CHECK-NEXT: 1 4 1.00 incp x0, p0.s +# CHECK-NEXT: 1 4 1.00 incp xzr, p15.b +# CHECK-NEXT: 1 4 1.00 incp xzr, p15.d +# CHECK-NEXT: 1 4 1.00 incp xzr, p15.h +# CHECK-NEXT: 1 4 1.00 incp xzr, p15.s # CHECK-NEXT: 1 4 0.50 incp z31.d, p15.d # CHECK-NEXT: 1 4 0.50 incp z31.h, p15.h # CHECK-NEXT: 1 4 0.50 incp z31.s, p15.s -# CHECK-NEXT: 1 1 0.33 incw x0 -# CHECK-NEXT: 1 1 0.33 incw x0, #14 -# CHECK-NEXT: 1 1 0.33 incw x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 incw x0, pow2 -# CHECK-NEXT: 1 1 0.33 incw x0, vl1 -# CHECK-NEXT: 1 4 0.50 incw z0.s -# CHECK-NEXT: 1 4 0.50 incw z0.s, all, mul #16 +# CHECK-NEXT: 1 3 0.33 incw x0 +# CHECK-NEXT: 1 3 0.33 incw x0, #14 +# CHECK-NEXT: 1 3 0.33 incw x0, all, mul #16 +# CHECK-NEXT: 1 3 0.33 incw x0, pow2 +# CHECK-NEXT: 1 3 0.33 incw x0, vl1 +# CHECK-NEXT: 1 3 0.50 incw z0.s +# CHECK-NEXT: 1 3 0.50 incw z0.s, all, mul #16 # CHECK-NEXT: 1 4 0.50 index z0.b, #0, #0 # CHECK-NEXT: 1 4 0.50 index z0.d, #0, #0 # CHECK-NEXT: 1 4 0.50 index z0.h, #0, #0 @@ -4412,8 +4412,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 * ld1b { z0.d }, p0/z, [x0] # CHECK-NEXT: 1 7 7.00 * ld1b { z0.d }, p0/z, [z0.d] # CHECK-NEXT: 1 3 0.50 * ld1b { z0.h }, p0/z, [x0] -# CHECK-NEXT: 1 9 4.50 * ld1b { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: 1 9 4.50 * ld1b { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: 1 7 3.50 * ld1b { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: 1 7 3.50 * ld1b { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: 1 3 0.50 * ld1b { z0.s }, p0/z, [x0] # CHECK-NEXT: 1 9 9.00 * ld1b { z0.s }, p0/z, [z0.s] # CHECK-NEXT: 1 3 0.50 * ld1b { z21.b }, p5/z, [x10, #5, mul vl] @@ -4450,8 +4450,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 * ld1h { z0.d }, p0/z, [x0] # CHECK-NEXT: 1 7 7.00 * ld1h { z0.d }, p0/z, [z0.d] # CHECK-NEXT: 1 3 0.50 * ld1h { z0.h }, p0/z, [x0] -# CHECK-NEXT: 1 9 4.50 * ld1h { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: 1 9 4.50 * ld1h { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: 1 7 3.50 * ld1h { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: 1 7 3.50 * ld1h { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: 1 3 0.50 * ld1h { z0.s }, p0/z, [x0] # CHECK-NEXT: 1 9 9.00 * ld1h { z0.s }, p0/z, [z0.s] # CHECK-NEXT: 1 3 0.50 * ld1h { z21.d }, p5/z, [x10, #5, mul vl] @@ -4467,8 +4467,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 7 7.00 * ld1h { z31.d }, p7/z, [z31.d, #62] # CHECK-NEXT: 1 3 0.50 * ld1h { z31.h }, p7/z, [sp, #-1, mul vl] # CHECK-NEXT: 1 3 0.50 * ld1h { z31.s }, p7/z, [sp, #-1, mul vl] -# CHECK-NEXT: 1 9 4.50 * ld1h { z31.s }, p7/z, [sp, z31.s, sxtw #1] -# CHECK-NEXT: 1 9 4.50 * ld1h { z31.s }, p7/z, [sp, z31.s, uxtw #1] +# CHECK-NEXT: 1 7 3.50 * ld1h { z31.s }, p7/z, [sp, z31.s, sxtw #1] +# CHECK-NEXT: 1 7 3.50 * ld1h { z31.s }, p7/z, [sp, z31.s, uxtw #1] # CHECK-NEXT: 1 9 9.00 * ld1h { z31.s }, p7/z, [z31.s, #62] # CHECK-NEXT: 1 3 0.50 * ld1h { z5.h }, p3/z, [sp, x16, lsl #1] # CHECK-NEXT: 1 3 0.50 * ld1h { z5.h }, p3/z, [x17, x16, lsl #1] @@ -4529,7 +4529,7 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 * ld1sb { z0.h }, p0/z, [sp, x0] # CHECK-NEXT: 1 3 0.50 * ld1sb { z0.h }, p0/z, [x0, x0] # CHECK-NEXT: 1 3 0.50 * ld1sb { z0.h }, p0/z, [x0] -# CHECK-NEXT: 1 9 4.50 * ld1sb { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: 1 7 3.50 * ld1sb { z0.s }, p0/z, [x0, z0.s, sxtw] # CHECK-NEXT: 1 3 0.50 * ld1sb { z0.s }, p0/z, [x0] # CHECK-NEXT: 1 9 9.00 * ld1sb { z0.s }, p0/z, [z0.s] # CHECK-NEXT: 1 3 0.50 * ld1sb { z21.d }, p5/z, [x10, #5, mul vl] @@ -4549,8 +4549,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 7 7.00 * ld1sh { z0.d }, p0/z, [x0, z0.d, uxtw #1] # CHECK-NEXT: 1 3 0.50 * ld1sh { z0.d }, p0/z, [x0] # CHECK-NEXT: 1 7 7.00 * ld1sh { z0.d }, p0/z, [z0.d] -# CHECK-NEXT: 1 9 4.50 * ld1sh { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: 1 9 4.50 * ld1sh { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: 1 7 3.50 * ld1sh { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: 1 7 3.50 * ld1sh { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: 1 3 0.50 * ld1sh { z0.s }, p0/z, [x0] # CHECK-NEXT: 1 9 9.00 * ld1sh { z0.s }, p0/z, [z0.s] # CHECK-NEXT: 1 3 0.50 * ld1sh { z21.d }, p5/z, [x10, #5, mul vl] @@ -4565,8 +4565,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 7 7.00 * ld1sh { z31.d }, p7/z, [sp, z31.d] # CHECK-NEXT: 1 7 7.00 * ld1sh { z31.d }, p7/z, [z31.d, #62] # CHECK-NEXT: 1 3 0.50 * ld1sh { z31.s }, p7/z, [sp, #-1, mul vl] -# CHECK-NEXT: 1 9 4.50 * ld1sh { z31.s }, p7/z, [sp, z31.s, sxtw #1] -# CHECK-NEXT: 1 9 4.50 * ld1sh { z31.s }, p7/z, [sp, z31.s, uxtw #1] +# CHECK-NEXT: 1 7 3.50 * ld1sh { z31.s }, p7/z, [sp, z31.s, sxtw #1] +# CHECK-NEXT: 1 7 3.50 * ld1sh { z31.s }, p7/z, [sp, z31.s, uxtw #1] # CHECK-NEXT: 1 9 9.00 * ld1sh { z31.s }, p7/z, [z31.s, #62] # CHECK-NEXT: 1 7 7.00 * ld1sw { z0.d }, p0/z, [x0, z0.d, sxtw #2] # CHECK-NEXT: 1 7 7.00 * ld1sw { z0.d }, p0/z, [x0, z0.d, uxtw #2] @@ -4585,8 +4585,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 7 7.00 * ld1w { z0.d }, p0/z, [x0, z0.d, uxtw #2] # CHECK-NEXT: 1 3 0.50 * ld1w { z0.d }, p0/z, [x0] # CHECK-NEXT: 1 7 7.00 * ld1w { z0.d }, p0/z, [z0.d] -# CHECK-NEXT: 1 9 4.50 * ld1w { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: 1 9 4.50 * ld1w { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: 1 7 3.50 * ld1w { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: 1 7 3.50 * ld1w { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: 1 3 0.50 * ld1w { z0.s }, p0/z, [x0] # CHECK-NEXT: 1 9 9.00 * ld1w { z0.s }, p0/z, [z0.s] # CHECK-NEXT: 1 3 0.50 * ld1w { z21.d }, p5/z, [x10, #5, mul vl] @@ -4601,8 +4601,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 7 7.00 * ld1w { z31.d }, p7/z, [sp, z31.d] # CHECK-NEXT: 1 7 7.00 * ld1w { z31.d }, p7/z, [z31.d, #124] # CHECK-NEXT: 1 3 0.50 * ld1w { z31.s }, p7/z, [sp, #-1, mul vl] -# CHECK-NEXT: 1 9 4.50 * ld1w { z31.s }, p7/z, [sp, z31.s, sxtw #2] -# CHECK-NEXT: 1 9 4.50 * ld1w { z31.s }, p7/z, [sp, z31.s, uxtw #2] +# CHECK-NEXT: 1 7 3.50 * ld1w { z31.s }, p7/z, [sp, z31.s, sxtw #2] +# CHECK-NEXT: 1 7 3.50 * ld1w { z31.s }, p7/z, [sp, z31.s, uxtw #2] # CHECK-NEXT: 1 9 9.00 * ld1w { z31.s }, p7/z, [z31.s, #124] # CHECK-NEXT: 1 3 2.00 * ld2b { z0.b, z1.b }, p0/z, [x0, x0] # CHECK-NEXT: 1 3 1.00 * ld2b { z0.b, z1.b }, p0/z, [x0] @@ -4668,8 +4668,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 7 7.00 * U ldff1b { z0.d }, p0/z, [z0.d] # CHECK-NEXT: 1 3 0.50 * U ldff1b { z0.h }, p0/z, [x0, x0] # CHECK-NEXT: 1 3 0.50 * U ldff1b { z0.s }, p0/z, [x0, x0] -# CHECK-NEXT: 1 9 4.50 * U ldff1b { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: 1 9 4.50 * U ldff1b { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: 1 7 3.50 * U ldff1b { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: 1 7 3.50 * U ldff1b { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: 1 9 9.00 * U ldff1b { z0.s }, p0/z, [z0.s] # CHECK-NEXT: 1 7 7.00 * U ldff1b { z21.d }, p5/z, [x10, z21.d, sxtw] # CHECK-NEXT: 1 7 7.00 * U ldff1b { z21.d }, p5/z, [x10, z21.d, uxtw] @@ -4696,8 +4696,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 7 7.00 * U ldff1h { z0.d }, p0/z, [z0.d] # CHECK-NEXT: 1 3 0.50 * U ldff1h { z0.h }, p0/z, [x0, x0, lsl #1] # CHECK-NEXT: 1 3 0.50 * U ldff1h { z0.s }, p0/z, [x0, x0, lsl #1] -# CHECK-NEXT: 1 9 4.50 * U ldff1h { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: 1 9 4.50 * U ldff1h { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: 1 7 3.50 * U ldff1h { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: 1 7 3.50 * U ldff1h { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: 1 9 9.00 * U ldff1h { z0.s }, p0/z, [z0.s] # CHECK-NEXT: 1 7 7.00 * U ldff1h { z21.d }, p5/z, [x10, z21.d, sxtw] # CHECK-NEXT: 1 7 7.00 * U ldff1h { z21.d }, p5/z, [x10, z21.d, uxtw] @@ -4706,16 +4706,16 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 * U ldff1h { z31.d }, p7/z, [sp] # CHECK-NEXT: 1 7 7.00 * U ldff1h { z31.d }, p7/z, [z31.d, #62] # CHECK-NEXT: 1 3 0.50 * U ldff1h { z31.h }, p7/z, [sp] -# CHECK-NEXT: 1 9 4.50 * U ldff1h { z31.s }, p7/z, [sp, z31.s, sxtw #1] -# CHECK-NEXT: 1 9 4.50 * U ldff1h { z31.s }, p7/z, [sp, z31.s, uxtw #1] +# CHECK-NEXT: 1 7 3.50 * U ldff1h { z31.s }, p7/z, [sp, z31.s, sxtw #1] +# CHECK-NEXT: 1 7 3.50 * U ldff1h { z31.s }, p7/z, [sp, z31.s, uxtw #1] # CHECK-NEXT: 1 3 0.50 * U ldff1h { z31.s }, p7/z, [sp] # CHECK-NEXT: 1 9 9.00 * U ldff1h { z31.s }, p7/z, [z31.s, #62] # CHECK-NEXT: 1 3 0.50 * U ldff1sb { z0.d }, p0/z, [x0, x0] # CHECK-NEXT: 1 7 7.00 * U ldff1sb { z0.d }, p0/z, [z0.d] # CHECK-NEXT: 1 3 0.50 * U ldff1sb { z0.h }, p0/z, [x0, x0] # CHECK-NEXT: 1 3 0.50 * U ldff1sb { z0.s }, p0/z, [x0, x0] -# CHECK-NEXT: 1 9 4.50 * U ldff1sb { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: 1 9 4.50 * U ldff1sb { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: 1 7 3.50 * U ldff1sb { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: 1 7 3.50 * U ldff1sb { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: 1 9 9.00 * U ldff1sb { z0.s }, p0/z, [z0.s] # CHECK-NEXT: 1 7 7.00 * U ldff1sb { z21.d }, p5/z, [x10, z21.d, sxtw] # CHECK-NEXT: 1 7 7.00 * U ldff1sb { z21.d }, p5/z, [x10, z21.d, uxtw] @@ -4730,8 +4730,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 7 7.00 * U ldff1sh { z0.d }, p0/z, [x0, z0.d, uxtw #1] # CHECK-NEXT: 1 7 7.00 * U ldff1sh { z0.d }, p0/z, [z0.d] # CHECK-NEXT: 1 3 0.50 * U ldff1sh { z0.s }, p0/z, [x0, x0, lsl #1] -# CHECK-NEXT: 1 9 4.50 * U ldff1sh { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: 1 9 4.50 * U ldff1sh { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: 1 7 3.50 * U ldff1sh { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: 1 7 3.50 * U ldff1sh { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: 1 9 9.00 * U ldff1sh { z0.s }, p0/z, [z0.s] # CHECK-NEXT: 1 7 7.00 * U ldff1sh { z21.d }, p5/z, [x10, z21.d, sxtw] # CHECK-NEXT: 1 7 7.00 * U ldff1sh { z21.d }, p5/z, [x10, z21.d, uxtw] @@ -4739,8 +4739,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 7 7.00 * U ldff1sh { z31.d }, p7/z, [sp, z31.d] # CHECK-NEXT: 1 3 0.50 * U ldff1sh { z31.d }, p7/z, [sp] # CHECK-NEXT: 1 7 7.00 * U ldff1sh { z31.d }, p7/z, [z31.d, #62] -# CHECK-NEXT: 1 9 4.50 * U ldff1sh { z31.s }, p7/z, [sp, z31.s, sxtw #1] -# CHECK-NEXT: 1 9 4.50 * U ldff1sh { z31.s }, p7/z, [sp, z31.s, uxtw #1] +# CHECK-NEXT: 1 7 3.50 * U ldff1sh { z31.s }, p7/z, [sp, z31.s, sxtw #1] +# CHECK-NEXT: 1 7 3.50 * U ldff1sh { z31.s }, p7/z, [sp, z31.s, uxtw #1] # CHECK-NEXT: 1 3 0.50 * U ldff1sh { z31.s }, p7/z, [sp] # CHECK-NEXT: 1 9 9.00 * U ldff1sh { z31.s }, p7/z, [z31.s, #62] # CHECK-NEXT: 1 3 0.50 * U ldff1sw { z0.d }, p0/z, [x0, x0, lsl #2] @@ -4758,8 +4758,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 7 7.00 * U ldff1w { z0.d }, p0/z, [x0, z0.d, uxtw #2] # CHECK-NEXT: 1 7 7.00 * U ldff1w { z0.d }, p0/z, [z0.d] # CHECK-NEXT: 1 3 0.50 * U ldff1w { z0.s }, p0/z, [x0, x0, lsl #2] -# CHECK-NEXT: 1 9 4.50 * U ldff1w { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: 1 9 4.50 * U ldff1w { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: 1 7 3.50 * U ldff1w { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: 1 7 3.50 * U ldff1w { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: 1 9 9.00 * U ldff1w { z0.s }, p0/z, [z0.s] # CHECK-NEXT: 1 7 7.00 * U ldff1w { z21.d }, p5/z, [x10, z21.d, sxtw] # CHECK-NEXT: 1 7 7.00 * U ldff1w { z21.d }, p5/z, [x10, z21.d, uxtw] @@ -4767,8 +4767,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 7 7.00 * U ldff1w { z31.d }, p7/z, [sp, z31.d] # CHECK-NEXT: 1 3 0.50 * U ldff1w { z31.d }, p7/z, [sp] # CHECK-NEXT: 1 7 7.00 * U ldff1w { z31.d }, p7/z, [z31.d, #124] -# CHECK-NEXT: 1 9 4.50 * U ldff1w { z31.s }, p7/z, [sp, z31.s, sxtw #2] -# CHECK-NEXT: 1 9 4.50 * U ldff1w { z31.s }, p7/z, [sp, z31.s, uxtw #2] +# CHECK-NEXT: 1 7 3.50 * U ldff1w { z31.s }, p7/z, [sp, z31.s, sxtw #2] +# CHECK-NEXT: 1 7 3.50 * U ldff1w { z31.s }, p7/z, [sp, z31.s, uxtw #2] # CHECK-NEXT: 1 3 0.50 * U ldff1w { z31.s }, p7/z, [sp] # CHECK-NEXT: 1 9 9.00 * U ldff1w { z31.s }, p7/z, [z31.s, #124] # CHECK-NEXT: 1 3 0.50 * U ldnf1b { z0.b }, p0/z, [x0] @@ -4959,12 +4959,12 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 4 0.50 mls z0.h, z1.h, z7.h[7] # CHECK-NEXT: 1 4 0.50 mls z0.s, p7/m, z1.s, z31.s # CHECK-NEXT: 1 4 0.50 mls z0.s, z1.s, z7.s[3] -# CHECK-NEXT: 1 6 1.00 mov p0.b, p0.b -# CHECK-NEXT: 1 6 1.00 mov p0.b, p0/m, p0.b -# CHECK-NEXT: 1 6 1.00 mov p0.b, p0/z, p0.b -# CHECK-NEXT: 1 6 1.00 mov p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 mov p15.b, p15/m, p15.b -# CHECK-NEXT: 1 6 1.00 mov p15.b, p15/z, p15.b +# CHECK-NEXT: 1 2 1.00 mov p0.b, p0.b +# CHECK-NEXT: 1 2 1.00 mov p0.b, p0/m, p0.b +# CHECK-NEXT: 1 2 1.00 mov p0.b, p0/z, p0.b +# CHECK-NEXT: 1 2 1.00 mov p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 mov p15.b, p15/m, p15.b +# CHECK-NEXT: 1 2 1.00 mov p15.b, p15/z, p15.b # CHECK-NEXT: 1 3 0.50 mov z0.b, #127 # CHECK-NEXT: 1 3 0.50 mov z0.b, b0 # CHECK-NEXT: 1 3 0.50 mov z0.b, p0/m, b0 @@ -5062,10 +5062,10 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 mov z5.h, #-6 # CHECK-NEXT: 1 3 0.50 mov z5.q, z17.q[3] # CHECK-NEXT: 1 3 0.50 mov z5.s, #-6 -# CHECK-NEXT: 1 6 1.00 movs p0.b, p0.b -# CHECK-NEXT: 1 6 1.00 movs p0.b, p0/z, p0.b -# CHECK-NEXT: 1 6 1.00 movs p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 movs p15.b, p15/z, p15.b +# CHECK-NEXT: 1 2 1.00 movs p0.b, p0.b +# CHECK-NEXT: 1 2 1.00 movs p0.b, p0/z, p0.b +# CHECK-NEXT: 1 2 1.00 movs p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 movs p15.b, p15/z, p15.b # CHECK-NEXT: 1 1 1.00 U mrs x3, ID_AA64ZFR0_EL1 # CHECK-NEXT: 1 1 1.00 U mrs x3, ZCR_EL1 # CHECK-NEXT: 1 1 1.00 U mrs x3, ZCR_EL12 @@ -5098,10 +5098,10 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 4 0.50 mul z31.h, z31.h, #127 # CHECK-NEXT: 1 4 0.50 mul z31.s, z31.s, #-128 # CHECK-NEXT: 1 4 0.50 mul z31.s, z31.s, #127 -# CHECK-NEXT: 1 6 1.00 nand p0.b, p0/z, p0.b, p0.b -# CHECK-NEXT: 1 6 1.00 nand p15.b, p15/z, p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 nands p0.b, p0/z, p0.b, p0.b -# CHECK-NEXT: 1 6 1.00 nands p15.b, p15/z, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 nand p0.b, p0/z, p0.b, p0.b +# CHECK-NEXT: 1 2 1.00 nand p15.b, p15/z, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 nands p0.b, p0/z, p0.b, p0.b +# CHECK-NEXT: 1 2 1.00 nands p15.b, p15/z, p15.b, p15.b # CHECK-NEXT: 1 3 0.50 nbsl z0.d, z0.d, z1.d, z2.d # CHECK-NEXT: 1 3 0.50 neg z0.b, p0/m, z0.b # CHECK-NEXT: 1 3 0.50 neg z0.d, p0/m, z0.d @@ -5115,23 +5115,23 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 7 1.00 nmatch p0.h, p0/z, z0.h, z0.h # CHECK-NEXT: 1 7 1.00 nmatch p15.b, p7/z, z30.b, z31.b # CHECK-NEXT: 1 7 1.00 nmatch p15.h, p7/z, z30.h, z31.h -# CHECK-NEXT: 1 6 1.00 nor p0.b, p0/z, p0.b, p0.b -# CHECK-NEXT: 1 6 1.00 nor p15.b, p15/z, p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 nors p0.b, p0/z, p0.b, p0.b -# CHECK-NEXT: 1 6 1.00 nors p15.b, p15/z, p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 not p0.b, p0/z, p0.b -# CHECK-NEXT: 1 6 1.00 not p15.b, p15/z, p15.b +# CHECK-NEXT: 1 2 1.00 nor p0.b, p0/z, p0.b, p0.b +# CHECK-NEXT: 1 2 1.00 nor p15.b, p15/z, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 nors p0.b, p0/z, p0.b, p0.b +# CHECK-NEXT: 1 2 1.00 nors p15.b, p15/z, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 not p0.b, p0/z, p0.b +# CHECK-NEXT: 1 2 1.00 not p15.b, p15/z, p15.b # CHECK-NEXT: 1 3 0.50 not z31.b, p7/m, z31.b # CHECK-NEXT: 1 3 0.50 not z31.d, p7/m, z31.d # CHECK-NEXT: 1 3 0.50 not z31.h, p7/m, z31.h # CHECK-NEXT: 1 3 0.50 not z31.s, p7/m, z31.s -# CHECK-NEXT: 1 6 1.00 nots p0.b, p0/z, p0.b -# CHECK-NEXT: 1 6 1.00 nots p15.b, p15/z, p15.b -# CHECK-NEXT: 1 6 1.00 orn p0.b, p0/z, p0.b, p0.b -# CHECK-NEXT: 1 6 1.00 orn p15.b, p15/z, p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 orns p0.b, p0/z, p0.b, p0.b -# CHECK-NEXT: 1 6 1.00 orns p15.b, p15/z, p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 orr p0.b, p0/z, p0.b, p1.b +# CHECK-NEXT: 1 2 1.00 nots p0.b, p0/z, p0.b +# CHECK-NEXT: 1 2 1.00 nots p15.b, p15/z, p15.b +# CHECK-NEXT: 1 2 1.00 orn p0.b, p0/z, p0.b, p0.b +# CHECK-NEXT: 1 2 1.00 orn p15.b, p15/z, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 orns p0.b, p0/z, p0.b, p0.b +# CHECK-NEXT: 1 2 1.00 orns p15.b, p15/z, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 orr p0.b, p0/z, p0.b, p1.b # CHECK-NEXT: 1 3 0.50 orr z0.d, z0.d, #0x6 # CHECK-NEXT: 1 3 0.50 orr z0.d, z0.d, #0xfffffffffffffff9 # CHECK-NEXT: 1 3 0.50 orr z0.s, z0.s, #0x6 @@ -5145,27 +5145,27 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 orr z31.s, p7/m, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 orr z5.b, z5.b, #0x6 # CHECK-NEXT: 1 3 0.50 orr z5.b, z5.b, #0xf9 -# CHECK-NEXT: 1 6 1.00 orrs p0.b, p0/z, p0.b, p1.b +# CHECK-NEXT: 1 2 1.00 orrs p0.b, p0/z, p0.b, p1.b # CHECK-NEXT: 1 4 1.00 orv b0, p7, z31.b # CHECK-NEXT: 1 4 1.00 orv d0, p7, z31.d # CHECK-NEXT: 1 4 1.00 orv h0, p7, z31.h # CHECK-NEXT: 1 4 1.00 orv s0, p7, z31.s -# CHECK-NEXT: 1 6 1.00 pfalse p15.b -# CHECK-NEXT: 1 6 1.00 pfirst p0.b, p15, p0.b -# CHECK-NEXT: 1 6 1.00 pfirst p15.b, p15, p15.b +# CHECK-NEXT: 1 2 1.00 pfalse p15.b +# CHECK-NEXT: 1 2 1.00 pfirst p0.b, p15, p0.b +# CHECK-NEXT: 1 2 1.00 pfirst p15.b, p15, p15.b # CHECK-NEXT: 1 4 0.50 pmul z0.b, z1.b, z2.b # CHECK-NEXT: 1 4 0.50 pmul z29.b, z30.b, z31.b -# CHECK-NEXT: 1 6 1.00 pmullb z0.h, z1.b, z2.b -# CHECK-NEXT: 1 6 1.00 pmullb z29.q, z30.d, z31.d -# CHECK-NEXT: 1 6 1.00 pmullb z31.d, z31.s, z31.s -# CHECK-NEXT: 1 6 1.00 pmullt z0.h, z1.b, z2.b -# CHECK-NEXT: 1 6 1.00 pmullt z29.q, z30.d, z31.d -# CHECK-NEXT: 1 6 1.00 pmullt z31.d, z31.s, z31.s -# CHECK-NEXT: 1 6 1.00 pnext p0.b, p15, p0.b -# CHECK-NEXT: 1 6 1.00 pnext p0.d, p15, p0.d -# CHECK-NEXT: 1 6 1.00 pnext p0.h, p15, p0.h -# CHECK-NEXT: 1 6 1.00 pnext p0.s, p15, p0.s -# CHECK-NEXT: 1 6 1.00 pnext p15.b, p15, p15.b +# CHECK-NEXT: 1 9 1.00 pmullb z0.h, z1.b, z2.b +# CHECK-NEXT: 1 9 1.00 pmullb z29.q, z30.d, z31.d +# CHECK-NEXT: 1 9 1.00 pmullb z31.d, z31.s, z31.s +# CHECK-NEXT: 1 9 1.00 pmullt z0.h, z1.b, z2.b +# CHECK-NEXT: 1 9 1.00 pmullt z29.q, z30.d, z31.d +# CHECK-NEXT: 1 9 1.00 pmullt z31.d, z31.s, z31.s +# CHECK-NEXT: 1 2 1.00 pnext p0.b, p15, p0.b +# CHECK-NEXT: 1 2 1.00 pnext p0.d, p15, p0.d +# CHECK-NEXT: 1 2 1.00 pnext p0.h, p15, p0.h +# CHECK-NEXT: 1 2 1.00 pnext p0.s, p15, p0.s +# CHECK-NEXT: 1 2 1.00 pnext p15.b, p15, p15.b # CHECK-NEXT: 1 0 0.50 * * U prfb #14, p0, [x0] # CHECK-NEXT: 1 0 0.50 * * U prfb #15, p0, [x0] # CHECK-NEXT: 1 0 0.50 * * U prfb #6, p0, [x0] @@ -5274,97 +5274,97 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 0 0.50 * * U prfw pstl2strm, p0, [x0] # CHECK-NEXT: 1 0 0.50 * * U prfw pstl3keep, p0, [x0] # CHECK-NEXT: 1 0 0.50 * * U prfw pstl3strm, p0, [x0] -# CHECK-NEXT: 1 6 1.00 ptest p15, p0.b -# CHECK-NEXT: 1 6 1.00 ptest p15, p15.b -# CHECK-NEXT: 1 6 1.00 ptrue p0.b, pow2 -# CHECK-NEXT: 1 6 1.00 ptrue p0.d, pow2 -# CHECK-NEXT: 1 6 1.00 ptrue p0.h, pow2 -# CHECK-NEXT: 1 6 1.00 ptrue p0.s, pow2 -# CHECK-NEXT: 1 6 1.00 ptrue p15.b -# CHECK-NEXT: 1 6 1.00 ptrue p15.d -# CHECK-NEXT: 1 6 1.00 ptrue p15.h -# CHECK-NEXT: 1 6 1.00 ptrue p15.s -# CHECK-NEXT: 1 6 1.00 ptrue p7.s -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, #14 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, #15 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, #16 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, #17 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, #18 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, #19 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, #20 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, #21 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, #22 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, #23 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, #24 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, #25 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, #26 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, #27 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, #28 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, mul3 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, mul4 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, vl1 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, vl128 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, vl16 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, vl2 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, vl256 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, vl3 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, vl32 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, vl4 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, vl5 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, vl6 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, vl64 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, vl7 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, vl8 -# CHECK-NEXT: 1 6 1.00 ptrues p0.b, pow2 -# CHECK-NEXT: 1 6 1.00 ptrues p0.d, pow2 -# CHECK-NEXT: 1 6 1.00 ptrues p0.h, pow2 -# CHECK-NEXT: 1 6 1.00 ptrues p0.s, pow2 -# CHECK-NEXT: 1 6 1.00 ptrues p15.b -# CHECK-NEXT: 1 6 1.00 ptrues p15.d -# CHECK-NEXT: 1 6 1.00 ptrues p15.h -# CHECK-NEXT: 1 6 1.00 ptrues p15.s -# CHECK-NEXT: 1 6 1.00 ptrues p7.s -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, #14 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, #15 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, #16 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, #17 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, #18 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, #19 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, #20 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, #21 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, #22 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, #23 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, #24 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, #25 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, #26 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, #27 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, #28 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, mul3 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, mul4 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, vl1 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, vl128 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, vl16 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, vl2 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, vl256 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, vl3 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, vl32 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, vl4 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, vl5 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, vl6 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, vl64 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, vl7 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, vl8 -# CHECK-NEXT: 1 6 1.00 punpkhi p0.h, p0.b -# CHECK-NEXT: 1 6 1.00 punpkhi p15.h, p15.b -# CHECK-NEXT: 1 6 1.00 punpklo p0.h, p0.b -# CHECK-NEXT: 1 6 1.00 punpklo p15.h, p15.b -# CHECK-NEXT: 1 4 0.50 raddhnb z0.b, z1.h, z31.h -# CHECK-NEXT: 1 4 0.50 raddhnb z0.h, z1.s, z31.s -# CHECK-NEXT: 1 4 0.50 raddhnb z0.s, z1.d, z31.d -# CHECK-NEXT: 1 4 0.50 raddhnt z0.b, z1.h, z31.h -# CHECK-NEXT: 1 4 0.50 raddhnt z0.h, z1.s, z31.s -# CHECK-NEXT: 1 4 0.50 raddhnt z0.s, z1.d, z31.d -# CHECK-NEXT: 1 8 1.00 rax1 z0.d, z1.d, z31.d +# CHECK-NEXT: 1 2 1.00 ptest p15, p0.b +# CHECK-NEXT: 1 2 1.00 ptest p15, p15.b +# CHECK-NEXT: 1 2 1.00 ptrue p0.b, pow2 +# CHECK-NEXT: 1 2 1.00 ptrue p0.d, pow2 +# CHECK-NEXT: 1 2 1.00 ptrue p0.h, pow2 +# CHECK-NEXT: 1 2 1.00 ptrue p0.s, pow2 +# CHECK-NEXT: 1 2 1.00 ptrue p15.b +# CHECK-NEXT: 1 2 1.00 ptrue p15.d +# CHECK-NEXT: 1 2 1.00 ptrue p15.h +# CHECK-NEXT: 1 2 1.00 ptrue p15.s +# CHECK-NEXT: 1 2 1.00 ptrue p7.s +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, #14 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, #15 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, #16 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, #17 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, #18 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, #19 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, #20 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, #21 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, #22 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, #23 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, #24 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, #25 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, #26 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, #27 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, #28 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, mul3 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, mul4 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, vl1 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, vl128 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, vl16 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, vl2 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, vl256 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, vl3 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, vl32 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, vl4 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, vl5 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, vl6 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, vl64 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, vl7 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, vl8 +# CHECK-NEXT: 1 2 1.00 ptrues p0.b, pow2 +# CHECK-NEXT: 1 2 1.00 ptrues p0.d, pow2 +# CHECK-NEXT: 1 2 1.00 ptrues p0.h, pow2 +# CHECK-NEXT: 1 2 1.00 ptrues p0.s, pow2 +# CHECK-NEXT: 1 2 1.00 ptrues p15.b +# CHECK-NEXT: 1 2 1.00 ptrues p15.d +# CHECK-NEXT: 1 2 1.00 ptrues p15.h +# CHECK-NEXT: 1 2 1.00 ptrues p15.s +# CHECK-NEXT: 1 2 1.00 ptrues p7.s +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, #14 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, #15 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, #16 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, #17 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, #18 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, #19 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, #20 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, #21 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, #22 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, #23 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, #24 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, #25 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, #26 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, #27 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, #28 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, mul3 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, mul4 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, vl1 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, vl128 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, vl16 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, vl2 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, vl256 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, vl3 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, vl32 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, vl4 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, vl5 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, vl6 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, vl64 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, vl7 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, vl8 +# CHECK-NEXT: 1 2 1.00 punpkhi p0.h, p0.b +# CHECK-NEXT: 1 2 1.00 punpkhi p15.h, p15.b +# CHECK-NEXT: 1 2 1.00 punpklo p0.h, p0.b +# CHECK-NEXT: 1 2 1.00 punpklo p15.h, p15.b +# CHECK-NEXT: 1 8 0.50 raddhnb z0.b, z1.h, z31.h +# CHECK-NEXT: 1 8 0.50 raddhnb z0.h, z1.s, z31.s +# CHECK-NEXT: 1 8 0.50 raddhnb z0.s, z1.d, z31.d +# CHECK-NEXT: 1 8 0.50 raddhnt z0.b, z1.h, z31.h +# CHECK-NEXT: 1 8 0.50 raddhnt z0.h, z1.s, z31.s +# CHECK-NEXT: 1 8 0.50 raddhnt z0.s, z1.d, z31.d +# CHECK-NEXT: 1 9 1.00 rax1 z0.d, z1.d, z31.d # CHECK-NEXT: 1 3 0.50 rbit z0.b, p7/m, z31.b # CHECK-NEXT: 1 3 0.50 rbit z0.d, p7/m, z31.d # CHECK-NEXT: 1 3 0.50 rbit z0.h, p7/m, z31.h @@ -5379,16 +5379,16 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 1 0.33 rdvl x21, #-32 # CHECK-NEXT: 1 1 0.33 rdvl x23, #31 # CHECK-NEXT: 1 1 0.33 rdvl xzr, #-1 -# CHECK-NEXT: 1 4 0.50 rev z0.b, z31.b -# CHECK-NEXT: 1 4 0.50 rev z0.d, z31.d -# CHECK-NEXT: 1 4 0.50 rev z0.h, z31.h -# CHECK-NEXT: 1 4 0.50 rev z0.s, z31.s -# CHECK-NEXT: 1 4 0.50 revb z0.d, p7/m, z31.d -# CHECK-NEXT: 1 4 0.50 revb z0.h, p7/m, z31.h -# CHECK-NEXT: 1 4 0.50 revb z0.s, p7/m, z31.s -# CHECK-NEXT: 1 4 0.50 revh z0.d, p7/m, z31.d -# CHECK-NEXT: 1 4 0.50 revh z0.s, p7/m, z31.s -# CHECK-NEXT: 1 4 0.50 revw z0.d, p7/m, z31.d +# CHECK-NEXT: 1 3 0.50 rev z0.b, z31.b +# CHECK-NEXT: 1 3 0.50 rev z0.d, z31.d +# CHECK-NEXT: 1 3 0.50 rev z0.h, z31.h +# CHECK-NEXT: 1 3 0.50 rev z0.s, z31.s +# CHECK-NEXT: 1 3 0.50 revb z0.d, p7/m, z31.d +# CHECK-NEXT: 1 3 0.50 revb z0.h, p7/m, z31.h +# CHECK-NEXT: 1 3 0.50 revb z0.s, p7/m, z31.s +# CHECK-NEXT: 1 3 0.50 revh z0.d, p7/m, z31.d +# CHECK-NEXT: 1 3 0.50 revh z0.s, p7/m, z31.s +# CHECK-NEXT: 1 3 0.50 revw z0.d, p7/m, z31.d # CHECK-NEXT: 1 4 0.50 rshrnb z0.b, z0.h, #1 # CHECK-NEXT: 1 4 0.50 rshrnb z0.h, z0.s, #1 # CHECK-NEXT: 1 4 0.50 rshrnb z0.s, z0.d, #1 @@ -5401,22 +5401,22 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 4 0.50 rshrnt z31.b, z31.h, #8 # CHECK-NEXT: 1 4 0.50 rshrnt z31.h, z31.s, #16 # CHECK-NEXT: 1 4 0.50 rshrnt z31.s, z31.d, #32 -# CHECK-NEXT: 1 4 0.50 rsubhnb z0.b, z1.h, z31.h -# CHECK-NEXT: 1 4 0.50 rsubhnb z0.h, z1.s, z31.s -# CHECK-NEXT: 1 4 0.50 rsubhnb z0.s, z1.d, z31.d -# CHECK-NEXT: 1 4 0.50 rsubhnt z0.b, z1.h, z31.h -# CHECK-NEXT: 1 4 0.50 rsubhnt z0.h, z1.s, z31.s -# CHECK-NEXT: 1 4 0.50 rsubhnt z0.s, z1.d, z31.d -# CHECK-NEXT: 1 8 1.00 saba z0.b, z1.b, z31.b -# CHECK-NEXT: 1 8 1.00 saba z0.d, z1.d, z31.d -# CHECK-NEXT: 1 8 1.00 saba z0.h, z1.h, z31.h -# CHECK-NEXT: 1 8 1.00 saba z0.s, z1.s, z31.s -# CHECK-NEXT: 1 8 1.00 sabalb z0.d, z1.s, z31.s -# CHECK-NEXT: 1 8 1.00 sabalb z0.h, z1.b, z31.b -# CHECK-NEXT: 1 8 1.00 sabalb z0.s, z1.h, z31.h -# CHECK-NEXT: 1 8 1.00 sabalt z0.d, z1.s, z31.s -# CHECK-NEXT: 1 8 1.00 sabalt z0.h, z1.b, z31.b -# CHECK-NEXT: 1 8 1.00 sabalt z0.s, z1.h, z31.h +# CHECK-NEXT: 1 8 0.50 rsubhnb z0.b, z1.h, z31.h +# CHECK-NEXT: 1 8 0.50 rsubhnb z0.h, z1.s, z31.s +# CHECK-NEXT: 1 8 0.50 rsubhnb z0.s, z1.d, z31.d +# CHECK-NEXT: 1 8 0.50 rsubhnt z0.b, z1.h, z31.h +# CHECK-NEXT: 1 8 0.50 rsubhnt z0.h, z1.s, z31.s +# CHECK-NEXT: 1 8 0.50 rsubhnt z0.s, z1.d, z31.d +# CHECK-NEXT: 1 6 1.00 saba z0.b, z1.b, z31.b +# CHECK-NEXT: 1 6 1.00 saba z0.d, z1.d, z31.d +# CHECK-NEXT: 1 6 1.00 saba z0.h, z1.h, z31.h +# CHECK-NEXT: 1 6 1.00 saba z0.s, z1.s, z31.s +# CHECK-NEXT: 1 6 1.00 sabalb z0.d, z1.s, z31.s +# CHECK-NEXT: 1 6 1.00 sabalb z0.h, z1.b, z31.b +# CHECK-NEXT: 1 6 1.00 sabalb z0.s, z1.h, z31.h +# CHECK-NEXT: 1 6 1.00 sabalt z0.d, z1.s, z31.s +# CHECK-NEXT: 1 6 1.00 sabalt z0.h, z1.b, z31.b +# CHECK-NEXT: 1 6 1.00 sabalt z0.s, z1.h, z31.h # CHECK-NEXT: 1 3 0.50 sabd z31.b, p7/m, z31.b, z31.b # CHECK-NEXT: 1 3 0.50 sabd z31.d, p7/m, z31.d, z31.d # CHECK-NEXT: 1 3 0.50 sabd z31.h, p7/m, z31.h, z31.h @@ -5430,24 +5430,24 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 7 1.00 sadalp z0.h, p0/m, z1.b # CHECK-NEXT: 1 7 1.00 sadalp z29.s, p0/m, z30.h # CHECK-NEXT: 1 7 1.00 sadalp z30.d, p7/m, z31.s -# CHECK-NEXT: 1 3 0.50 saddlb z0.h, z1.b, z2.b -# CHECK-NEXT: 1 3 0.50 saddlb z29.s, z30.h, z31.h -# CHECK-NEXT: 1 3 0.50 saddlb z31.d, z31.s, z31.s -# CHECK-NEXT: 1 3 0.50 saddlbt z0.d, z1.s, z31.s -# CHECK-NEXT: 1 3 0.50 saddlbt z0.h, z1.b, z31.b -# CHECK-NEXT: 1 3 0.50 saddlbt z0.s, z1.h, z31.h -# CHECK-NEXT: 1 3 0.50 saddlt z0.h, z1.b, z2.b -# CHECK-NEXT: 1 3 0.50 saddlt z29.s, z30.h, z31.h -# CHECK-NEXT: 1 3 0.50 saddlt z31.d, z31.s, z31.s +# CHECK-NEXT: 1 4 0.50 saddlb z0.h, z1.b, z2.b +# CHECK-NEXT: 1 4 0.50 saddlb z29.s, z30.h, z31.h +# CHECK-NEXT: 1 4 0.50 saddlb z31.d, z31.s, z31.s +# CHECK-NEXT: 1 4 0.50 saddlbt z0.d, z1.s, z31.s +# CHECK-NEXT: 1 4 0.50 saddlbt z0.h, z1.b, z31.b +# CHECK-NEXT: 1 4 0.50 saddlbt z0.s, z1.h, z31.h +# CHECK-NEXT: 1 4 0.50 saddlt z0.h, z1.b, z2.b +# CHECK-NEXT: 1 4 0.50 saddlt z29.s, z30.h, z31.h +# CHECK-NEXT: 1 4 0.50 saddlt z31.d, z31.s, z31.s # CHECK-NEXT: 1 4 1.00 saddv d0, p7, z31.b # CHECK-NEXT: 1 4 1.00 saddv d0, p7, z31.h # CHECK-NEXT: 1 4 1.00 saddv d0, p7, z31.s -# CHECK-NEXT: 1 3 0.50 saddwb z0.h, z1.h, z2.b -# CHECK-NEXT: 1 3 0.50 saddwb z29.s, z30.s, z31.h -# CHECK-NEXT: 1 3 0.50 saddwb z31.d, z31.d, z31.s -# CHECK-NEXT: 1 3 0.50 saddwt z0.h, z1.h, z2.b -# CHECK-NEXT: 1 3 0.50 saddwt z29.s, z30.s, z31.h -# CHECK-NEXT: 1 3 0.50 saddwt z31.d, z31.d, z31.s +# CHECK-NEXT: 1 4 0.50 saddwb z0.h, z1.h, z2.b +# CHECK-NEXT: 1 4 0.50 saddwb z29.s, z30.s, z31.h +# CHECK-NEXT: 1 4 0.50 saddwb z31.d, z31.d, z31.s +# CHECK-NEXT: 1 4 0.50 saddwt z0.h, z1.h, z2.b +# CHECK-NEXT: 1 4 0.50 saddwt z29.s, z30.s, z31.h +# CHECK-NEXT: 1 4 0.50 saddwt z31.d, z31.d, z31.s # CHECK-NEXT: 1 4 0.50 sbclb z0.d, z1.d, z31.d # CHECK-NEXT: 1 4 0.50 sbclb z0.s, z1.s, z31.s # CHECK-NEXT: 1 4 0.50 sbclt z0.d, z1.d, z31.d @@ -5504,8 +5504,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 sli z31.d, z31.d, #63 # CHECK-NEXT: 1 3 0.50 sli z31.h, z31.h, #15 # CHECK-NEXT: 1 3 0.50 sli z31.s, z31.s, #31 -# CHECK-NEXT: 1 8 1.00 sm4e z0.s, z0.s, z31.s -# CHECK-NEXT: 1 8 1.00 sm4ekey z0.s, z1.s, z31.s +# CHECK-NEXT: 1 9 1.00 sm4e z0.s, z0.s, z31.s +# CHECK-NEXT: 1 9 1.00 sm4ekey z0.s, z1.s, z31.s # CHECK-NEXT: 1 3 0.50 smax z0.b, z0.b, #-128 # CHECK-NEXT: 1 3 0.50 smax z0.d, z0.d, #-128 # CHECK-NEXT: 1 3 0.50 smax z0.h, z0.h, #-128 @@ -5624,61 +5624,61 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 4 0.50 sqcadd z31.d, z31.d, z31.d, #270 # CHECK-NEXT: 1 4 0.50 sqcadd z31.h, z31.h, z31.h, #270 # CHECK-NEXT: 1 4 0.50 sqcadd z31.s, z31.s, z31.s, #270 -# CHECK-NEXT: 1 1 0.33 sqdecb x0 -# CHECK-NEXT: 1 1 0.33 sqdecb x0, #14 -# CHECK-NEXT: 1 1 0.33 sqdecb x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 sqdecb x0, pow2 -# CHECK-NEXT: 1 1 0.33 sqdecb x0, vl1 -# CHECK-NEXT: 1 1 0.33 sqdecb x0, w0 -# CHECK-NEXT: 1 1 0.33 sqdecb x0, w0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 sqdecb x0, w0, pow2 -# CHECK-NEXT: 1 1 0.33 sqdecb x0, w0, pow2, mul #16 -# CHECK-NEXT: 1 1 0.33 sqdecd x0 -# CHECK-NEXT: 1 1 0.33 sqdecd x0, #14 -# CHECK-NEXT: 1 1 0.33 sqdecd x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 sqdecd x0, pow2 -# CHECK-NEXT: 1 1 0.33 sqdecd x0, vl1 -# CHECK-NEXT: 1 1 0.33 sqdecd x0, w0 -# CHECK-NEXT: 1 1 0.33 sqdecd x0, w0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 sqdecd x0, w0, pow2 -# CHECK-NEXT: 1 1 0.33 sqdecd x0, w0, pow2, mul #16 +# CHECK-NEXT: 1 4 0.33 sqdecb x0 +# CHECK-NEXT: 1 4 0.33 sqdecb x0, #14 +# CHECK-NEXT: 1 4 0.33 sqdecb x0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 sqdecb x0, pow2 +# CHECK-NEXT: 1 4 0.33 sqdecb x0, vl1 +# CHECK-NEXT: 1 4 0.33 sqdecb x0, w0 +# CHECK-NEXT: 1 4 0.33 sqdecb x0, w0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 sqdecb x0, w0, pow2 +# CHECK-NEXT: 1 4 0.33 sqdecb x0, w0, pow2, mul #16 +# CHECK-NEXT: 1 4 0.33 sqdecd x0 +# CHECK-NEXT: 1 4 0.33 sqdecd x0, #14 +# CHECK-NEXT: 1 4 0.33 sqdecd x0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 sqdecd x0, pow2 +# CHECK-NEXT: 1 4 0.33 sqdecd x0, vl1 +# CHECK-NEXT: 1 4 0.33 sqdecd x0, w0 +# CHECK-NEXT: 1 4 0.33 sqdecd x0, w0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 sqdecd x0, w0, pow2 +# CHECK-NEXT: 1 4 0.33 sqdecd x0, w0, pow2, mul #16 # CHECK-NEXT: 1 4 0.50 sqdecd z0.d # CHECK-NEXT: 1 4 0.50 sqdecd z0.d, all, mul #16 # CHECK-NEXT: 1 4 0.50 sqdecd z0.d, pow2 # CHECK-NEXT: 1 4 0.50 sqdecd z0.d, pow2, mul #16 -# CHECK-NEXT: 1 1 0.33 sqdech x0 -# CHECK-NEXT: 1 1 0.33 sqdech x0, #14 -# CHECK-NEXT: 1 1 0.33 sqdech x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 sqdech x0, pow2 -# CHECK-NEXT: 1 1 0.33 sqdech x0, vl1 -# CHECK-NEXT: 1 1 0.33 sqdech x0, w0 -# CHECK-NEXT: 1 1 0.33 sqdech x0, w0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 sqdech x0, w0, pow2 -# CHECK-NEXT: 1 1 0.33 sqdech x0, w0, pow2, mul #16 +# CHECK-NEXT: 1 4 0.33 sqdech x0 +# CHECK-NEXT: 1 4 0.33 sqdech x0, #14 +# CHECK-NEXT: 1 4 0.33 sqdech x0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 sqdech x0, pow2 +# CHECK-NEXT: 1 4 0.33 sqdech x0, vl1 +# CHECK-NEXT: 1 4 0.33 sqdech x0, w0 +# CHECK-NEXT: 1 4 0.33 sqdech x0, w0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 sqdech x0, w0, pow2 +# CHECK-NEXT: 1 4 0.33 sqdech x0, w0, pow2, mul #16 # CHECK-NEXT: 1 4 0.50 sqdech z0.h # CHECK-NEXT: 1 4 0.50 sqdech z0.h, all, mul #16 # CHECK-NEXT: 1 4 0.50 sqdech z0.h, pow2 # CHECK-NEXT: 1 4 0.50 sqdech z0.h, pow2, mul #16 -# CHECK-NEXT: 1 8 1.00 sqdecp x0, p0.b -# CHECK-NEXT: 1 8 1.00 sqdecp x0, p0.d -# CHECK-NEXT: 1 8 1.00 sqdecp x0, p0.h -# CHECK-NEXT: 1 8 1.00 sqdecp x0, p0.s -# CHECK-NEXT: 1 8 1.00 sqdecp xzr, p15.b, wzr -# CHECK-NEXT: 1 8 1.00 sqdecp xzr, p15.d, wzr -# CHECK-NEXT: 1 8 1.00 sqdecp xzr, p15.h, wzr -# CHECK-NEXT: 1 8 1.00 sqdecp xzr, p15.s, wzr +# CHECK-NEXT: 1 9 1.00 sqdecp x0, p0.b +# CHECK-NEXT: 1 9 1.00 sqdecp x0, p0.d +# CHECK-NEXT: 1 9 1.00 sqdecp x0, p0.h +# CHECK-NEXT: 1 9 1.00 sqdecp x0, p0.s +# CHECK-NEXT: 1 9 1.00 sqdecp xzr, p15.b, wzr +# CHECK-NEXT: 1 9 1.00 sqdecp xzr, p15.d, wzr +# CHECK-NEXT: 1 9 1.00 sqdecp xzr, p15.h, wzr +# CHECK-NEXT: 1 9 1.00 sqdecp xzr, p15.s, wzr # CHECK-NEXT: 1 4 0.50 sqdecp z0.d, p0.d # CHECK-NEXT: 1 4 0.50 sqdecp z0.h, p0.h # CHECK-NEXT: 1 4 0.50 sqdecp z0.s, p0.s -# CHECK-NEXT: 1 1 0.33 sqdecw x0 -# CHECK-NEXT: 1 1 0.33 sqdecw x0, #14 -# CHECK-NEXT: 1 1 0.33 sqdecw x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 sqdecw x0, pow2 -# CHECK-NEXT: 1 1 0.33 sqdecw x0, vl1 -# CHECK-NEXT: 1 1 0.33 sqdecw x0, w0 -# CHECK-NEXT: 1 1 0.33 sqdecw x0, w0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 sqdecw x0, w0, pow2 -# CHECK-NEXT: 1 1 0.33 sqdecw x0, w0, pow2, mul #16 +# CHECK-NEXT: 1 4 0.33 sqdecw x0 +# CHECK-NEXT: 1 4 0.33 sqdecw x0, #14 +# CHECK-NEXT: 1 4 0.33 sqdecw x0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 sqdecw x0, pow2 +# CHECK-NEXT: 1 4 0.33 sqdecw x0, vl1 +# CHECK-NEXT: 1 4 0.33 sqdecw x0, w0 +# CHECK-NEXT: 1 4 0.33 sqdecw x0, w0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 sqdecw x0, w0, pow2 +# CHECK-NEXT: 1 4 0.33 sqdecw x0, w0, pow2, mul #16 # CHECK-NEXT: 1 4 0.50 sqdecw z0.s # CHECK-NEXT: 1 4 0.50 sqdecw z0.s, all, mul #16 # CHECK-NEXT: 1 4 0.50 sqdecw z0.s, pow2 @@ -5726,61 +5726,61 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 4 0.50 sqdmullt z0.s, z1.h, z7.h[7] # CHECK-NEXT: 1 4 0.50 sqdmullt z29.s, z30.h, z31.h # CHECK-NEXT: 1 4 0.50 sqdmullt z31.d, z31.s, z31.s -# CHECK-NEXT: 1 1 0.33 sqincb x0 -# CHECK-NEXT: 1 1 0.33 sqincb x0, #14 -# CHECK-NEXT: 1 1 0.33 sqincb x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 sqincb x0, pow2 -# CHECK-NEXT: 1 1 0.33 sqincb x0, vl1 -# CHECK-NEXT: 1 1 0.33 sqincb x0, w0 -# CHECK-NEXT: 1 1 0.33 sqincb x0, w0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 sqincb x0, w0, pow2 -# CHECK-NEXT: 1 1 0.33 sqincb x0, w0, pow2, mul #16 -# CHECK-NEXT: 1 1 0.33 sqincd x0 -# CHECK-NEXT: 1 1 0.33 sqincd x0, #14 -# CHECK-NEXT: 1 1 0.33 sqincd x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 sqincd x0, pow2 -# CHECK-NEXT: 1 1 0.33 sqincd x0, vl1 -# CHECK-NEXT: 1 1 0.33 sqincd x0, w0 -# CHECK-NEXT: 1 1 0.33 sqincd x0, w0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 sqincd x0, w0, pow2 -# CHECK-NEXT: 1 1 0.33 sqincd x0, w0, pow2, mul #16 +# CHECK-NEXT: 1 4 0.33 sqincb x0 +# CHECK-NEXT: 1 4 0.33 sqincb x0, #14 +# CHECK-NEXT: 1 4 0.33 sqincb x0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 sqincb x0, pow2 +# CHECK-NEXT: 1 4 0.33 sqincb x0, vl1 +# CHECK-NEXT: 1 4 0.33 sqincb x0, w0 +# CHECK-NEXT: 1 4 0.33 sqincb x0, w0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 sqincb x0, w0, pow2 +# CHECK-NEXT: 1 4 0.33 sqincb x0, w0, pow2, mul #16 +# CHECK-NEXT: 1 4 0.33 sqincd x0 +# CHECK-NEXT: 1 4 0.33 sqincd x0, #14 +# CHECK-NEXT: 1 4 0.33 sqincd x0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 sqincd x0, pow2 +# CHECK-NEXT: 1 4 0.33 sqincd x0, vl1 +# CHECK-NEXT: 1 4 0.33 sqincd x0, w0 +# CHECK-NEXT: 1 4 0.33 sqincd x0, w0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 sqincd x0, w0, pow2 +# CHECK-NEXT: 1 4 0.33 sqincd x0, w0, pow2, mul #16 # CHECK-NEXT: 1 4 0.50 sqincd z0.d # CHECK-NEXT: 1 4 0.50 sqincd z0.d, all, mul #16 # CHECK-NEXT: 1 4 0.50 sqincd z0.d, pow2 # CHECK-NEXT: 1 4 0.50 sqincd z0.d, pow2, mul #16 -# CHECK-NEXT: 1 1 0.33 sqinch x0 -# CHECK-NEXT: 1 1 0.33 sqinch x0, #14 -# CHECK-NEXT: 1 1 0.33 sqinch x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 sqinch x0, pow2 -# CHECK-NEXT: 1 1 0.33 sqinch x0, vl1 -# CHECK-NEXT: 1 1 0.33 sqinch x0, w0 -# CHECK-NEXT: 1 1 0.33 sqinch x0, w0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 sqinch x0, w0, pow2 -# CHECK-NEXT: 1 1 0.33 sqinch x0, w0, pow2, mul #16 +# CHECK-NEXT: 1 4 0.33 sqinch x0 +# CHECK-NEXT: 1 4 0.33 sqinch x0, #14 +# CHECK-NEXT: 1 4 0.33 sqinch x0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 sqinch x0, pow2 +# CHECK-NEXT: 1 4 0.33 sqinch x0, vl1 +# CHECK-NEXT: 1 4 0.33 sqinch x0, w0 +# CHECK-NEXT: 1 4 0.33 sqinch x0, w0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 sqinch x0, w0, pow2 +# CHECK-NEXT: 1 4 0.33 sqinch x0, w0, pow2, mul #16 # CHECK-NEXT: 1 4 0.50 sqinch z0.h # CHECK-NEXT: 1 4 0.50 sqinch z0.h, all, mul #16 # CHECK-NEXT: 1 4 0.50 sqinch z0.h, pow2 # CHECK-NEXT: 1 4 0.50 sqinch z0.h, pow2, mul #16 -# CHECK-NEXT: 1 8 1.00 sqincp x0, p0.b -# CHECK-NEXT: 1 8 1.00 sqincp x0, p0.d -# CHECK-NEXT: 1 8 1.00 sqincp x0, p0.h -# CHECK-NEXT: 1 8 1.00 sqincp x0, p0.s -# CHECK-NEXT: 1 8 1.00 sqincp xzr, p15.b, wzr -# CHECK-NEXT: 1 8 1.00 sqincp xzr, p15.d, wzr -# CHECK-NEXT: 1 8 1.00 sqincp xzr, p15.h, wzr -# CHECK-NEXT: 1 8 1.00 sqincp xzr, p15.s, wzr +# CHECK-NEXT: 1 9 1.00 sqincp x0, p0.b +# CHECK-NEXT: 1 9 1.00 sqincp x0, p0.d +# CHECK-NEXT: 1 9 1.00 sqincp x0, p0.h +# CHECK-NEXT: 1 9 1.00 sqincp x0, p0.s +# CHECK-NEXT: 1 9 1.00 sqincp xzr, p15.b, wzr +# CHECK-NEXT: 1 9 1.00 sqincp xzr, p15.d, wzr +# CHECK-NEXT: 1 9 1.00 sqincp xzr, p15.h, wzr +# CHECK-NEXT: 1 9 1.00 sqincp xzr, p15.s, wzr # CHECK-NEXT: 1 4 0.50 sqincp z0.d, p0.d # CHECK-NEXT: 1 4 0.50 sqincp z0.h, p0.h # CHECK-NEXT: 1 4 0.50 sqincp z0.s, p0.s -# CHECK-NEXT: 1 1 0.33 sqincw x0 -# CHECK-NEXT: 1 1 0.33 sqincw x0, #14 -# CHECK-NEXT: 1 1 0.33 sqincw x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 sqincw x0, pow2 -# CHECK-NEXT: 1 1 0.33 sqincw x0, vl1 -# CHECK-NEXT: 1 1 0.33 sqincw x0, w0 -# CHECK-NEXT: 1 1 0.33 sqincw x0, w0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 sqincw x0, w0, pow2 -# CHECK-NEXT: 1 1 0.33 sqincw x0, w0, pow2, mul #16 +# CHECK-NEXT: 1 4 0.33 sqincw x0 +# CHECK-NEXT: 1 4 0.33 sqincw x0, #14 +# CHECK-NEXT: 1 4 0.33 sqincw x0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 sqincw x0, pow2 +# CHECK-NEXT: 1 4 0.33 sqincw x0, vl1 +# CHECK-NEXT: 1 4 0.33 sqincw x0, w0 +# CHECK-NEXT: 1 4 0.33 sqincw x0, w0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 sqincw x0, w0, pow2 +# CHECK-NEXT: 1 4 0.33 sqincw x0, w0, pow2, mul #16 # CHECK-NEXT: 1 4 0.50 sqincw z0.s # CHECK-NEXT: 1 4 0.50 sqincw z0.s, all, mul #16 # CHECK-NEXT: 1 4 0.50 sqincw z0.s, pow2 @@ -6001,24 +6001,24 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 4 0.50 ssra z31.d, z31.d, #64 # CHECK-NEXT: 1 4 0.50 ssra z31.h, z31.h, #16 # CHECK-NEXT: 1 4 0.50 ssra z31.s, z31.s, #32 -# CHECK-NEXT: 1 3 0.50 ssublb z0.h, z1.b, z2.b -# CHECK-NEXT: 1 3 0.50 ssublb z29.s, z30.h, z31.h -# CHECK-NEXT: 1 3 0.50 ssublb z31.d, z31.s, z31.s -# CHECK-NEXT: 1 3 0.50 ssublbt z0.d, z1.s, z31.s -# CHECK-NEXT: 1 3 0.50 ssublbt z0.h, z1.b, z31.b -# CHECK-NEXT: 1 3 0.50 ssublbt z0.s, z1.h, z31.h -# CHECK-NEXT: 1 3 0.50 ssublt z0.h, z1.b, z2.b -# CHECK-NEXT: 1 3 0.50 ssublt z29.s, z30.h, z31.h -# CHECK-NEXT: 1 3 0.50 ssublt z31.d, z31.s, z31.s -# CHECK-NEXT: 1 3 0.50 ssubltb z0.d, z1.s, z31.s -# CHECK-NEXT: 1 3 0.50 ssubltb z0.h, z1.b, z31.b -# CHECK-NEXT: 1 3 0.50 ssubltb z0.s, z1.h, z31.h -# CHECK-NEXT: 1 3 0.50 ssubwb z0.h, z1.h, z2.b -# CHECK-NEXT: 1 3 0.50 ssubwb z29.s, z30.s, z31.h -# CHECK-NEXT: 1 3 0.50 ssubwb z31.d, z31.d, z31.s -# CHECK-NEXT: 1 3 0.50 ssubwt z0.h, z1.h, z2.b -# CHECK-NEXT: 1 3 0.50 ssubwt z29.s, z30.s, z31.h -# CHECK-NEXT: 1 3 0.50 ssubwt z31.d, z31.d, z31.s +# CHECK-NEXT: 1 4 0.50 ssublb z0.h, z1.b, z2.b +# CHECK-NEXT: 1 4 0.50 ssublb z29.s, z30.h, z31.h +# CHECK-NEXT: 1 4 0.50 ssublb z31.d, z31.s, z31.s +# CHECK-NEXT: 1 4 0.50 ssublbt z0.d, z1.s, z31.s +# CHECK-NEXT: 1 4 0.50 ssublbt z0.h, z1.b, z31.b +# CHECK-NEXT: 1 4 0.50 ssublbt z0.s, z1.h, z31.h +# CHECK-NEXT: 1 4 0.50 ssublt z0.h, z1.b, z2.b +# CHECK-NEXT: 1 4 0.50 ssublt z29.s, z30.h, z31.h +# CHECK-NEXT: 1 4 0.50 ssublt z31.d, z31.s, z31.s +# CHECK-NEXT: 1 4 0.50 ssubltb z0.d, z1.s, z31.s +# CHECK-NEXT: 1 4 0.50 ssubltb z0.h, z1.b, z31.b +# CHECK-NEXT: 1 4 0.50 ssubltb z0.s, z1.h, z31.h +# CHECK-NEXT: 1 4 0.50 ssubwb z0.h, z1.h, z2.b +# CHECK-NEXT: 1 4 0.50 ssubwb z29.s, z30.s, z31.h +# CHECK-NEXT: 1 4 0.50 ssubwb z31.d, z31.d, z31.s +# CHECK-NEXT: 1 4 0.50 ssubwt z0.h, z1.h, z2.b +# CHECK-NEXT: 1 4 0.50 ssubwt z29.s, z30.s, z31.h +# CHECK-NEXT: 1 4 0.50 ssubwt z31.d, z31.d, z31.s # CHECK-NEXT: 1 1 1.00 * st1b { z0.b }, p0, [x0, x0] # CHECK-NEXT: 1 1 1.00 * st1b { z0.b }, p0, [x0] # CHECK-NEXT: 1 1 1.00 * st1b { z0.d }, p0, [x0, x0] @@ -6250,12 +6250,12 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 sub z31.s, p7/m, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 sub z31.s, z31.s, #65280 # CHECK-NEXT: 1 3 0.50 sub z31.s, z31.s, z31.s -# CHECK-NEXT: 1 4 0.50 subhnb z0.b, z1.h, z31.h -# CHECK-NEXT: 1 4 0.50 subhnb z0.h, z1.s, z31.s -# CHECK-NEXT: 1 4 0.50 subhnb z0.s, z1.d, z31.d -# CHECK-NEXT: 1 4 0.50 subhnt z0.b, z1.h, z31.h -# CHECK-NEXT: 1 4 0.50 subhnt z0.h, z1.s, z31.s -# CHECK-NEXT: 1 4 0.50 subhnt z0.s, z1.d, z31.d +# CHECK-NEXT: 1 8 0.50 subhnb z0.b, z1.h, z31.h +# CHECK-NEXT: 1 8 0.50 subhnb z0.h, z1.s, z31.s +# CHECK-NEXT: 1 8 0.50 subhnb z0.s, z1.d, z31.d +# CHECK-NEXT: 1 8 0.50 subhnt z0.b, z1.h, z31.h +# CHECK-NEXT: 1 8 0.50 subhnt z0.h, z1.s, z31.s +# CHECK-NEXT: 1 8 0.50 subhnt z0.s, z1.d, z31.d # CHECK-NEXT: 1 3 0.50 subr z0.b, p0/m, z0.b, z0.b # CHECK-NEXT: 1 3 0.50 subr z0.b, z0.b, #0 # CHECK-NEXT: 1 3 0.50 subr z0.d, p0/m, z0.d, z0.d @@ -6305,32 +6305,32 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 4 0.50 tbx z31.d, z31.d, z31.d # CHECK-NEXT: 1 4 0.50 tbx z31.h, z31.h, z31.h # CHECK-NEXT: 1 4 0.50 tbx z31.s, z31.s, z31.s -# CHECK-NEXT: 1 6 1.00 trn1 p15.b, p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 trn1 p15.d, p15.d, p15.d -# CHECK-NEXT: 1 6 1.00 trn1 p15.h, p15.h, p15.h -# CHECK-NEXT: 1 6 1.00 trn1 p15.s, p15.s, p15.s -# CHECK-NEXT: 1 4 0.50 trn1 z31.b, z31.b, z31.b -# CHECK-NEXT: 1 4 0.50 trn1 z31.d, z31.d, z31.d -# CHECK-NEXT: 1 4 0.50 trn1 z31.h, z31.h, z31.h -# CHECK-NEXT: 1 4 0.50 trn1 z31.s, z31.s, z31.s -# CHECK-NEXT: 1 6 1.00 trn2 p15.b, p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 trn2 p15.d, p15.d, p15.d -# CHECK-NEXT: 1 6 1.00 trn2 p15.h, p15.h, p15.h -# CHECK-NEXT: 1 6 1.00 trn2 p15.s, p15.s, p15.s -# CHECK-NEXT: 1 4 0.50 trn2 z31.b, z31.b, z31.b -# CHECK-NEXT: 1 4 0.50 trn2 z31.d, z31.d, z31.d -# CHECK-NEXT: 1 4 0.50 trn2 z31.h, z31.h, z31.h -# CHECK-NEXT: 1 4 0.50 trn2 z31.s, z31.s, z31.s -# CHECK-NEXT: 1 8 1.00 uaba z0.b, z1.b, z31.b -# CHECK-NEXT: 1 8 1.00 uaba z0.d, z1.d, z31.d -# CHECK-NEXT: 1 8 1.00 uaba z0.h, z1.h, z31.h -# CHECK-NEXT: 1 8 1.00 uaba z0.s, z1.s, z31.s -# CHECK-NEXT: 1 8 1.00 uabalb z0.d, z1.s, z31.s -# CHECK-NEXT: 1 8 1.00 uabalb z0.h, z1.b, z31.b -# CHECK-NEXT: 1 8 1.00 uabalb z0.s, z1.h, z31.h -# CHECK-NEXT: 1 8 1.00 uabalt z0.d, z1.s, z31.s -# CHECK-NEXT: 1 8 1.00 uabalt z0.h, z1.b, z31.b -# CHECK-NEXT: 1 8 1.00 uabalt z0.s, z1.h, z31.h +# CHECK-NEXT: 1 2 1.00 trn1 p15.b, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 trn1 p15.d, p15.d, p15.d +# CHECK-NEXT: 1 2 1.00 trn1 p15.h, p15.h, p15.h +# CHECK-NEXT: 1 2 1.00 trn1 p15.s, p15.s, p15.s +# CHECK-NEXT: 1 3 0.50 trn1 z31.b, z31.b, z31.b +# CHECK-NEXT: 1 3 0.50 trn1 z31.d, z31.d, z31.d +# CHECK-NEXT: 1 3 0.50 trn1 z31.h, z31.h, z31.h +# CHECK-NEXT: 1 3 0.50 trn1 z31.s, z31.s, z31.s +# CHECK-NEXT: 1 2 1.00 trn2 p15.b, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 trn2 p15.d, p15.d, p15.d +# CHECK-NEXT: 1 2 1.00 trn2 p15.h, p15.h, p15.h +# CHECK-NEXT: 1 2 1.00 trn2 p15.s, p15.s, p15.s +# CHECK-NEXT: 1 3 0.50 trn2 z31.b, z31.b, z31.b +# CHECK-NEXT: 1 3 0.50 trn2 z31.d, z31.d, z31.d +# CHECK-NEXT: 1 3 0.50 trn2 z31.h, z31.h, z31.h +# CHECK-NEXT: 1 3 0.50 trn2 z31.s, z31.s, z31.s +# CHECK-NEXT: 1 6 1.00 uaba z0.b, z1.b, z31.b +# CHECK-NEXT: 1 6 1.00 uaba z0.d, z1.d, z31.d +# CHECK-NEXT: 1 6 1.00 uaba z0.h, z1.h, z31.h +# CHECK-NEXT: 1 6 1.00 uaba z0.s, z1.s, z31.s +# CHECK-NEXT: 1 6 1.00 uabalb z0.d, z1.s, z31.s +# CHECK-NEXT: 1 6 1.00 uabalb z0.h, z1.b, z31.b +# CHECK-NEXT: 1 6 1.00 uabalb z0.s, z1.h, z31.h +# CHECK-NEXT: 1 6 1.00 uabalt z0.d, z1.s, z31.s +# CHECK-NEXT: 1 6 1.00 uabalt z0.h, z1.b, z31.b +# CHECK-NEXT: 1 6 1.00 uabalt z0.s, z1.h, z31.h # CHECK-NEXT: 1 3 0.50 uabd z31.b, p7/m, z31.b, z31.b # CHECK-NEXT: 1 3 0.50 uabd z31.d, p7/m, z31.d, z31.d # CHECK-NEXT: 1 3 0.50 uabd z31.h, p7/m, z31.h, z31.h @@ -6344,22 +6344,22 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 7 1.00 uadalp z0.h, p0/m, z1.b # CHECK-NEXT: 1 7 1.00 uadalp z29.s, p0/m, z30.h # CHECK-NEXT: 1 7 1.00 uadalp z30.d, p7/m, z31.s -# CHECK-NEXT: 1 3 0.50 uaddlb z0.h, z1.b, z2.b -# CHECK-NEXT: 1 3 0.50 uaddlb z29.s, z30.h, z31.h -# CHECK-NEXT: 1 3 0.50 uaddlb z31.d, z31.s, z31.s -# CHECK-NEXT: 1 3 0.50 uaddlt z0.h, z1.b, z2.b -# CHECK-NEXT: 1 3 0.50 uaddlt z29.s, z30.h, z31.h -# CHECK-NEXT: 1 3 0.50 uaddlt z31.d, z31.s, z31.s +# CHECK-NEXT: 1 4 0.50 uaddlb z0.h, z1.b, z2.b +# CHECK-NEXT: 1 4 0.50 uaddlb z29.s, z30.h, z31.h +# CHECK-NEXT: 1 4 0.50 uaddlb z31.d, z31.s, z31.s +# CHECK-NEXT: 1 4 0.50 uaddlt z0.h, z1.b, z2.b +# CHECK-NEXT: 1 4 0.50 uaddlt z29.s, z30.h, z31.h +# CHECK-NEXT: 1 4 0.50 uaddlt z31.d, z31.s, z31.s # CHECK-NEXT: 1 4 1.00 uaddv d0, p7, z31.b # CHECK-NEXT: 1 4 1.00 uaddv d0, p7, z31.d # CHECK-NEXT: 1 4 1.00 uaddv d0, p7, z31.h # CHECK-NEXT: 1 4 1.00 uaddv d0, p7, z31.s -# CHECK-NEXT: 1 3 0.50 uaddwb z0.h, z1.h, z2.b -# CHECK-NEXT: 1 3 0.50 uaddwb z29.s, z30.s, z31.h -# CHECK-NEXT: 1 3 0.50 uaddwb z31.d, z31.d, z31.s -# CHECK-NEXT: 1 3 0.50 uaddwt z0.h, z1.h, z2.b -# CHECK-NEXT: 1 3 0.50 uaddwt z29.s, z30.s, z31.h -# CHECK-NEXT: 1 3 0.50 uaddwt z31.d, z31.d, z31.s +# CHECK-NEXT: 1 4 0.50 uaddwb z0.h, z1.h, z2.b +# CHECK-NEXT: 1 4 0.50 uaddwb z29.s, z30.s, z31.h +# CHECK-NEXT: 1 4 0.50 uaddwb z31.d, z31.d, z31.s +# CHECK-NEXT: 1 4 0.50 uaddwt z0.h, z1.h, z2.b +# CHECK-NEXT: 1 4 0.50 uaddwt z29.s, z30.s, z31.h +# CHECK-NEXT: 1 4 0.50 uaddwt z31.d, z31.d, z31.s # CHECK-NEXT: 1 4 0.50 ucvtf z0.d, p0/m, z0.d # CHECK-NEXT: 1 4 0.50 ucvtf z0.d, p0/m, z0.s # CHECK-NEXT: 1 4 0.50 ucvtf z0.h, p0/m, z0.d @@ -6473,120 +6473,120 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 4 0.50 uqadd z31.d, z31.d, #65280 # CHECK-NEXT: 1 4 0.50 uqadd z31.h, z31.h, #65280 # CHECK-NEXT: 1 4 0.50 uqadd z31.s, z31.s, #65280 -# CHECK-NEXT: 1 1 0.33 uqdecb w0 -# CHECK-NEXT: 1 1 0.33 uqdecb w0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 uqdecb w0, pow2 -# CHECK-NEXT: 1 1 0.33 uqdecb w0, pow2, mul #16 -# CHECK-NEXT: 1 1 0.33 uqdecb x0 -# CHECK-NEXT: 1 1 0.33 uqdecb x0, #14 -# CHECK-NEXT: 1 1 0.33 uqdecb x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 uqdecb x0, pow2 -# CHECK-NEXT: 1 1 0.33 uqdecb x0, vl1 -# CHECK-NEXT: 1 1 0.33 uqdecd w0 -# CHECK-NEXT: 1 1 0.33 uqdecd w0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 uqdecd w0, pow2 -# CHECK-NEXT: 1 1 0.33 uqdecd w0, pow2, mul #16 -# CHECK-NEXT: 1 1 0.33 uqdecd x0 -# CHECK-NEXT: 1 1 0.33 uqdecd x0, #14 -# CHECK-NEXT: 1 1 0.33 uqdecd x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 uqdecd x0, pow2 -# CHECK-NEXT: 1 1 0.33 uqdecd x0, vl1 +# CHECK-NEXT: 1 4 0.33 uqdecb w0 +# CHECK-NEXT: 1 4 0.33 uqdecb w0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 uqdecb w0, pow2 +# CHECK-NEXT: 1 4 0.33 uqdecb w0, pow2, mul #16 +# CHECK-NEXT: 1 4 0.33 uqdecb x0 +# CHECK-NEXT: 1 4 0.33 uqdecb x0, #14 +# CHECK-NEXT: 1 4 0.33 uqdecb x0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 uqdecb x0, pow2 +# CHECK-NEXT: 1 4 0.33 uqdecb x0, vl1 +# CHECK-NEXT: 1 4 0.33 uqdecd w0 +# CHECK-NEXT: 1 4 0.33 uqdecd w0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 uqdecd w0, pow2 +# CHECK-NEXT: 1 4 0.33 uqdecd w0, pow2, mul #16 +# CHECK-NEXT: 1 4 0.33 uqdecd x0 +# CHECK-NEXT: 1 4 0.33 uqdecd x0, #14 +# CHECK-NEXT: 1 4 0.33 uqdecd x0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 uqdecd x0, pow2 +# CHECK-NEXT: 1 4 0.33 uqdecd x0, vl1 # CHECK-NEXT: 1 4 0.50 uqdecd z0.d # CHECK-NEXT: 1 4 0.50 uqdecd z0.d, all, mul #16 # CHECK-NEXT: 1 4 0.50 uqdecd z0.d, pow2 # CHECK-NEXT: 1 4 0.50 uqdecd z0.d, pow2, mul #16 -# CHECK-NEXT: 1 1 0.33 uqdech w0 -# CHECK-NEXT: 1 1 0.33 uqdech w0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 uqdech w0, pow2 -# CHECK-NEXT: 1 1 0.33 uqdech w0, pow2, mul #16 -# CHECK-NEXT: 1 1 0.33 uqdech x0 -# CHECK-NEXT: 1 1 0.33 uqdech x0, #14 -# CHECK-NEXT: 1 1 0.33 uqdech x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 uqdech x0, pow2 -# CHECK-NEXT: 1 1 0.33 uqdech x0, vl1 +# CHECK-NEXT: 1 4 0.33 uqdech w0 +# CHECK-NEXT: 1 4 0.33 uqdech w0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 uqdech w0, pow2 +# CHECK-NEXT: 1 4 0.33 uqdech w0, pow2, mul #16 +# CHECK-NEXT: 1 4 0.33 uqdech x0 +# CHECK-NEXT: 1 4 0.33 uqdech x0, #14 +# CHECK-NEXT: 1 4 0.33 uqdech x0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 uqdech x0, pow2 +# CHECK-NEXT: 1 4 0.33 uqdech x0, vl1 # CHECK-NEXT: 1 4 0.50 uqdech z0.h # CHECK-NEXT: 1 4 0.50 uqdech z0.h, all, mul #16 # CHECK-NEXT: 1 4 0.50 uqdech z0.h, pow2 # CHECK-NEXT: 1 4 0.50 uqdech z0.h, pow2, mul #16 -# CHECK-NEXT: 1 8 1.00 uqdecp wzr, p15.b -# CHECK-NEXT: 1 8 1.00 uqdecp wzr, p15.d -# CHECK-NEXT: 1 8 1.00 uqdecp wzr, p15.h -# CHECK-NEXT: 1 8 1.00 uqdecp wzr, p15.s -# CHECK-NEXT: 1 8 1.00 uqdecp x0, p0.b -# CHECK-NEXT: 1 8 1.00 uqdecp x0, p0.d -# CHECK-NEXT: 1 8 1.00 uqdecp x0, p0.h -# CHECK-NEXT: 1 8 1.00 uqdecp x0, p0.s +# CHECK-NEXT: 1 9 1.00 uqdecp wzr, p15.b +# CHECK-NEXT: 1 9 1.00 uqdecp wzr, p15.d +# CHECK-NEXT: 1 9 1.00 uqdecp wzr, p15.h +# CHECK-NEXT: 1 9 1.00 uqdecp wzr, p15.s +# CHECK-NEXT: 1 9 1.00 uqdecp x0, p0.b +# CHECK-NEXT: 1 9 1.00 uqdecp x0, p0.d +# CHECK-NEXT: 1 9 1.00 uqdecp x0, p0.h +# CHECK-NEXT: 1 9 1.00 uqdecp x0, p0.s # CHECK-NEXT: 1 4 0.50 uqdecp z0.d, p0.d # CHECK-NEXT: 1 4 0.50 uqdecp z0.h, p0.h # CHECK-NEXT: 1 4 0.50 uqdecp z0.s, p0.s -# CHECK-NEXT: 1 1 0.33 uqdecw w0 -# CHECK-NEXT: 1 1 0.33 uqdecw w0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 uqdecw w0, pow2 -# CHECK-NEXT: 1 1 0.33 uqdecw w0, pow2, mul #16 -# CHECK-NEXT: 1 1 0.33 uqdecw x0 -# CHECK-NEXT: 1 1 0.33 uqdecw x0, #14 -# CHECK-NEXT: 1 1 0.33 uqdecw x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 uqdecw x0, pow2 -# CHECK-NEXT: 1 1 0.33 uqdecw x0, vl1 +# CHECK-NEXT: 1 4 0.33 uqdecw w0 +# CHECK-NEXT: 1 4 0.33 uqdecw w0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 uqdecw w0, pow2 +# CHECK-NEXT: 1 4 0.33 uqdecw w0, pow2, mul #16 +# CHECK-NEXT: 1 4 0.33 uqdecw x0 +# CHECK-NEXT: 1 4 0.33 uqdecw x0, #14 +# CHECK-NEXT: 1 4 0.33 uqdecw x0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 uqdecw x0, pow2 +# CHECK-NEXT: 1 4 0.33 uqdecw x0, vl1 # CHECK-NEXT: 1 4 0.50 uqdecw z0.s # CHECK-NEXT: 1 4 0.50 uqdecw z0.s, all, mul #16 # CHECK-NEXT: 1 4 0.50 uqdecw z0.s, pow2 # CHECK-NEXT: 1 4 0.50 uqdecw z0.s, pow2, mul #16 -# CHECK-NEXT: 1 1 0.33 uqincb w0 -# CHECK-NEXT: 1 1 0.33 uqincb w0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 uqincb w0, pow2 -# CHECK-NEXT: 1 1 0.33 uqincb w0, pow2, mul #16 -# CHECK-NEXT: 1 1 0.33 uqincb x0 -# CHECK-NEXT: 1 1 0.33 uqincb x0, #14 -# CHECK-NEXT: 1 1 0.33 uqincb x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 uqincb x0, pow2 -# CHECK-NEXT: 1 1 0.33 uqincb x0, vl1 -# CHECK-NEXT: 1 1 0.33 uqincd w0 -# CHECK-NEXT: 1 1 0.33 uqincd w0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 uqincd w0, pow2 -# CHECK-NEXT: 1 1 0.33 uqincd w0, pow2, mul #16 -# CHECK-NEXT: 1 1 0.33 uqincd x0 -# CHECK-NEXT: 1 1 0.33 uqincd x0, #14 -# CHECK-NEXT: 1 1 0.33 uqincd x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 uqincd x0, pow2 -# CHECK-NEXT: 1 1 0.33 uqincd x0, vl1 +# CHECK-NEXT: 1 4 0.33 uqincb w0 +# CHECK-NEXT: 1 4 0.33 uqincb w0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 uqincb w0, pow2 +# CHECK-NEXT: 1 4 0.33 uqincb w0, pow2, mul #16 +# CHECK-NEXT: 1 4 0.33 uqincb x0 +# CHECK-NEXT: 1 4 0.33 uqincb x0, #14 +# CHECK-NEXT: 1 4 0.33 uqincb x0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 uqincb x0, pow2 +# CHECK-NEXT: 1 4 0.33 uqincb x0, vl1 +# CHECK-NEXT: 1 4 0.33 uqincd w0 +# CHECK-NEXT: 1 4 0.33 uqincd w0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 uqincd w0, pow2 +# CHECK-NEXT: 1 4 0.33 uqincd w0, pow2, mul #16 +# CHECK-NEXT: 1 4 0.33 uqincd x0 +# CHECK-NEXT: 1 4 0.33 uqincd x0, #14 +# CHECK-NEXT: 1 4 0.33 uqincd x0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 uqincd x0, pow2 +# CHECK-NEXT: 1 4 0.33 uqincd x0, vl1 # CHECK-NEXT: 1 4 0.50 uqincd z0.d # CHECK-NEXT: 1 4 0.50 uqincd z0.d, all, mul #16 # CHECK-NEXT: 1 4 0.50 uqincd z0.d, pow2 # CHECK-NEXT: 1 4 0.50 uqincd z0.d, pow2, mul #16 -# CHECK-NEXT: 1 1 0.33 uqinch w0 -# CHECK-NEXT: 1 1 0.33 uqinch w0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 uqinch w0, pow2 -# CHECK-NEXT: 1 1 0.33 uqinch w0, pow2, mul #16 -# CHECK-NEXT: 1 1 0.33 uqinch x0 -# CHECK-NEXT: 1 1 0.33 uqinch x0, #14 -# CHECK-NEXT: 1 1 0.33 uqinch x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 uqinch x0, pow2 -# CHECK-NEXT: 1 1 0.33 uqinch x0, vl1 +# CHECK-NEXT: 1 4 0.33 uqinch w0 +# CHECK-NEXT: 1 4 0.33 uqinch w0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 uqinch w0, pow2 +# CHECK-NEXT: 1 4 0.33 uqinch w0, pow2, mul #16 +# CHECK-NEXT: 1 4 0.33 uqinch x0 +# CHECK-NEXT: 1 4 0.33 uqinch x0, #14 +# CHECK-NEXT: 1 4 0.33 uqinch x0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 uqinch x0, pow2 +# CHECK-NEXT: 1 4 0.33 uqinch x0, vl1 # CHECK-NEXT: 1 4 0.50 uqinch z0.h # CHECK-NEXT: 1 4 0.50 uqinch z0.h, all, mul #16 # CHECK-NEXT: 1 4 0.50 uqinch z0.h, pow2 # CHECK-NEXT: 1 4 0.50 uqinch z0.h, pow2, mul #16 -# CHECK-NEXT: 1 8 1.00 uqincp wzr, p15.b -# CHECK-NEXT: 1 8 1.00 uqincp wzr, p15.d -# CHECK-NEXT: 1 8 1.00 uqincp wzr, p15.h -# CHECK-NEXT: 1 8 1.00 uqincp wzr, p15.s -# CHECK-NEXT: 1 8 1.00 uqincp x0, p0.b -# CHECK-NEXT: 1 8 1.00 uqincp x0, p0.d -# CHECK-NEXT: 1 8 1.00 uqincp x0, p0.h -# CHECK-NEXT: 1 8 1.00 uqincp x0, p0.s +# CHECK-NEXT: 1 9 1.00 uqincp wzr, p15.b +# CHECK-NEXT: 1 9 1.00 uqincp wzr, p15.d +# CHECK-NEXT: 1 9 1.00 uqincp wzr, p15.h +# CHECK-NEXT: 1 9 1.00 uqincp wzr, p15.s +# CHECK-NEXT: 1 9 1.00 uqincp x0, p0.b +# CHECK-NEXT: 1 9 1.00 uqincp x0, p0.d +# CHECK-NEXT: 1 9 1.00 uqincp x0, p0.h +# CHECK-NEXT: 1 9 1.00 uqincp x0, p0.s # CHECK-NEXT: 1 4 0.50 uqincp z0.d, p0.d # CHECK-NEXT: 1 4 0.50 uqincp z0.h, p0.h # CHECK-NEXT: 1 4 0.50 uqincp z0.s, p0.s -# CHECK-NEXT: 1 1 0.33 uqincw w0 -# CHECK-NEXT: 1 1 0.33 uqincw w0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 uqincw w0, pow2 -# CHECK-NEXT: 1 1 0.33 uqincw w0, pow2, mul #16 -# CHECK-NEXT: 1 1 0.33 uqincw x0 -# CHECK-NEXT: 1 1 0.33 uqincw x0, #14 -# CHECK-NEXT: 1 1 0.33 uqincw x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 uqincw x0, pow2 -# CHECK-NEXT: 1 1 0.33 uqincw x0, vl1 +# CHECK-NEXT: 1 4 0.33 uqincw w0 +# CHECK-NEXT: 1 4 0.33 uqincw w0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 uqincw w0, pow2 +# CHECK-NEXT: 1 4 0.33 uqincw w0, pow2, mul #16 +# CHECK-NEXT: 1 4 0.33 uqincw x0 +# CHECK-NEXT: 1 4 0.33 uqincw x0, #14 +# CHECK-NEXT: 1 4 0.33 uqincw x0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 uqincw x0, pow2 +# CHECK-NEXT: 1 4 0.33 uqincw x0, vl1 # CHECK-NEXT: 1 4 0.50 uqincw z0.s # CHECK-NEXT: 1 4 0.50 uqincw z0.s, all, mul #16 # CHECK-NEXT: 1 4 0.50 uqincw z0.s, pow2 @@ -6723,18 +6723,18 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 4 0.50 usra z31.d, z31.d, #64 # CHECK-NEXT: 1 4 0.50 usra z31.h, z31.h, #16 # CHECK-NEXT: 1 4 0.50 usra z31.s, z31.s, #32 -# CHECK-NEXT: 1 3 0.50 usublb z0.h, z1.b, z2.b -# CHECK-NEXT: 1 3 0.50 usublb z29.s, z30.h, z31.h -# CHECK-NEXT: 1 3 0.50 usublb z31.d, z31.s, z31.s -# CHECK-NEXT: 1 3 0.50 usublt z0.h, z1.b, z2.b -# CHECK-NEXT: 1 3 0.50 usublt z29.s, z30.h, z31.h -# CHECK-NEXT: 1 3 0.50 usublt z31.d, z31.s, z31.s -# CHECK-NEXT: 1 3 0.50 usubwb z0.h, z1.h, z2.b -# CHECK-NEXT: 1 3 0.50 usubwb z29.s, z30.s, z31.h -# CHECK-NEXT: 1 3 0.50 usubwb z31.d, z31.d, z31.s -# CHECK-NEXT: 1 3 0.50 usubwt z0.h, z1.h, z2.b -# CHECK-NEXT: 1 3 0.50 usubwt z29.s, z30.s, z31.h -# CHECK-NEXT: 1 3 0.50 usubwt z31.d, z31.d, z31.s +# CHECK-NEXT: 1 4 0.50 usublb z0.h, z1.b, z2.b +# CHECK-NEXT: 1 4 0.50 usublb z29.s, z30.h, z31.h +# CHECK-NEXT: 1 4 0.50 usublb z31.d, z31.s, z31.s +# CHECK-NEXT: 1 4 0.50 usublt z0.h, z1.b, z2.b +# CHECK-NEXT: 1 4 0.50 usublt z29.s, z30.h, z31.h +# CHECK-NEXT: 1 4 0.50 usublt z31.d, z31.s, z31.s +# CHECK-NEXT: 1 4 0.50 usubwb z0.h, z1.h, z2.b +# CHECK-NEXT: 1 4 0.50 usubwb z29.s, z30.s, z31.h +# CHECK-NEXT: 1 4 0.50 usubwb z31.d, z31.d, z31.s +# CHECK-NEXT: 1 4 0.50 usubwt z0.h, z1.h, z2.b +# CHECK-NEXT: 1 4 0.50 usubwt z29.s, z30.s, z31.h +# CHECK-NEXT: 1 4 0.50 usubwt z31.d, z31.d, z31.s # CHECK-NEXT: 1 4 0.50 uunpkhi z31.d, z31.s # CHECK-NEXT: 1 4 0.50 uunpkhi z31.h, z31.b # CHECK-NEXT: 1 4 0.50 uunpkhi z31.s, z31.h @@ -6753,82 +6753,82 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 uxth z31.s, p7/m, z31.s # CHECK-NEXT: 1 3 0.50 uxtw z0.d, p0/m, z0.d # CHECK-NEXT: 1 3 0.50 uxtw z31.d, p7/m, z31.d -# CHECK-NEXT: 1 6 1.00 uzp1 p15.b, p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 uzp1 p15.d, p15.d, p15.d -# CHECK-NEXT: 1 6 1.00 uzp1 p15.h, p15.h, p15.h -# CHECK-NEXT: 1 6 1.00 uzp1 p15.s, p15.s, p15.s -# CHECK-NEXT: 1 4 0.50 uzp1 z31.b, z31.b, z31.b -# CHECK-NEXT: 1 4 0.50 uzp1 z31.d, z31.d, z31.d -# CHECK-NEXT: 1 4 0.50 uzp1 z31.h, z31.h, z31.h -# CHECK-NEXT: 1 4 0.50 uzp1 z31.s, z31.s, z31.s -# CHECK-NEXT: 1 6 1.00 uzp2 p15.b, p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 uzp2 p15.d, p15.d, p15.d -# CHECK-NEXT: 1 6 1.00 uzp2 p15.h, p15.h, p15.h -# CHECK-NEXT: 1 6 1.00 uzp2 p15.s, p15.s, p15.s -# CHECK-NEXT: 1 4 0.50 uzp2 z31.b, z31.b, z31.b -# CHECK-NEXT: 1 4 0.50 uzp2 z31.d, z31.d, z31.d -# CHECK-NEXT: 1 4 0.50 uzp2 z31.h, z31.h, z31.h -# CHECK-NEXT: 1 4 0.50 uzp2 z31.s, z31.s, z31.s -# CHECK-NEXT: 1 6 1.00 whilege p15.b, w0, wzr -# CHECK-NEXT: 1 6 1.00 whilege p15.b, wzr, w0 -# CHECK-NEXT: 1 6 1.00 whilege p15.b, x0, xzr -# CHECK-NEXT: 1 6 1.00 whilege p15.b, xzr, x0 -# CHECK-NEXT: 1 6 1.00 whilege p15.d, w0, wzr -# CHECK-NEXT: 1 6 1.00 whilege p15.d, x0, xzr -# CHECK-NEXT: 1 6 1.00 whilege p15.h, w0, wzr -# CHECK-NEXT: 1 6 1.00 whilege p15.h, x0, xzr -# CHECK-NEXT: 1 6 1.00 whilege p15.s, w0, wzr -# CHECK-NEXT: 1 6 1.00 whilege p15.s, x0, xzr -# CHECK-NEXT: 1 6 1.00 whilerw p15.b, x30, x30 -# CHECK-NEXT: 1 6 1.00 whilerw p15.d, x30, x30 -# CHECK-NEXT: 1 6 1.00 whilerw p15.h, x30, x30 -# CHECK-NEXT: 1 6 1.00 whilerw p15.s, x30, x30 -# CHECK-NEXT: 1 6 1.00 whilewr p15.b, x30, x30 -# CHECK-NEXT: 1 6 1.00 whilewr p15.d, x30, x30 -# CHECK-NEXT: 1 6 1.00 whilewr p15.h, x30, x30 -# CHECK-NEXT: 1 6 1.00 whilewr p15.s, x30, x30 +# CHECK-NEXT: 1 2 1.00 uzp1 p15.b, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 uzp1 p15.d, p15.d, p15.d +# CHECK-NEXT: 1 2 1.00 uzp1 p15.h, p15.h, p15.h +# CHECK-NEXT: 1 2 1.00 uzp1 p15.s, p15.s, p15.s +# CHECK-NEXT: 1 3 0.50 uzp1 z31.b, z31.b, z31.b +# CHECK-NEXT: 1 3 0.50 uzp1 z31.d, z31.d, z31.d +# CHECK-NEXT: 1 3 0.50 uzp1 z31.h, z31.h, z31.h +# CHECK-NEXT: 1 3 0.50 uzp1 z31.s, z31.s, z31.s +# CHECK-NEXT: 1 2 1.00 uzp2 p15.b, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 uzp2 p15.d, p15.d, p15.d +# CHECK-NEXT: 1 2 1.00 uzp2 p15.h, p15.h, p15.h +# CHECK-NEXT: 1 2 1.00 uzp2 p15.s, p15.s, p15.s +# CHECK-NEXT: 1 3 0.50 uzp2 z31.b, z31.b, z31.b +# CHECK-NEXT: 1 3 0.50 uzp2 z31.d, z31.d, z31.d +# CHECK-NEXT: 1 3 0.50 uzp2 z31.h, z31.h, z31.h +# CHECK-NEXT: 1 3 0.50 uzp2 z31.s, z31.s, z31.s +# CHECK-NEXT: 1 2 1.00 whilege p15.b, w0, wzr +# CHECK-NEXT: 1 2 1.00 whilege p15.b, wzr, w0 +# CHECK-NEXT: 1 2 1.00 whilege p15.b, x0, xzr +# CHECK-NEXT: 1 2 1.00 whilege p15.b, xzr, x0 +# CHECK-NEXT: 1 2 1.00 whilege p15.d, w0, wzr +# CHECK-NEXT: 1 2 1.00 whilege p15.d, x0, xzr +# CHECK-NEXT: 1 2 1.00 whilege p15.h, w0, wzr +# CHECK-NEXT: 1 2 1.00 whilege p15.h, x0, xzr +# CHECK-NEXT: 1 2 1.00 whilege p15.s, w0, wzr +# CHECK-NEXT: 1 2 1.00 whilege p15.s, x0, xzr +# CHECK-NEXT: 1 2 1.00 whilerw p15.b, x30, x30 +# CHECK-NEXT: 1 2 1.00 whilerw p15.d, x30, x30 +# CHECK-NEXT: 1 2 1.00 whilerw p15.h, x30, x30 +# CHECK-NEXT: 1 2 1.00 whilerw p15.s, x30, x30 +# CHECK-NEXT: 1 2 1.00 whilewr p15.b, x30, x30 +# CHECK-NEXT: 1 2 1.00 whilewr p15.d, x30, x30 +# CHECK-NEXT: 1 2 1.00 whilewr p15.h, x30, x30 +# CHECK-NEXT: 1 2 1.00 whilewr p15.s, x30, x30 # CHECK-NEXT: 1 1 0.33 * U wrffr p0.b # CHECK-NEXT: 1 1 0.33 * U wrffr p15.b -# CHECK-NEXT: 1 3 0.50 xar z0.b, z0.b, z1.b, #1 -# CHECK-NEXT: 1 3 0.50 xar z0.d, z0.d, z1.d, #1 -# CHECK-NEXT: 1 3 0.50 xar z0.h, z0.h, z1.h, #1 -# CHECK-NEXT: 1 3 0.50 xar z0.s, z0.s, z1.s, #1 -# CHECK-NEXT: 1 3 0.50 xar z31.b, z31.b, z30.b, #8 -# CHECK-NEXT: 1 3 0.50 xar z31.d, z31.d, z30.d, #64 -# CHECK-NEXT: 1 3 0.50 xar z31.h, z31.h, z30.h, #16 -# CHECK-NEXT: 1 3 0.50 xar z31.s, z31.s, z30.s, #32 -# CHECK-NEXT: 1 6 1.00 zip1 p0.b, p0.b, p0.b -# CHECK-NEXT: 1 6 1.00 zip1 p0.d, p0.d, p0.d -# CHECK-NEXT: 1 6 1.00 zip1 p0.h, p0.h, p0.h -# CHECK-NEXT: 1 6 1.00 zip1 p0.s, p0.s, p0.s -# CHECK-NEXT: 1 6 1.00 zip1 p15.b, p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 zip1 p15.d, p15.d, p15.d -# CHECK-NEXT: 1 6 1.00 zip1 p15.h, p15.h, p15.h -# CHECK-NEXT: 1 6 1.00 zip1 p15.s, p15.s, p15.s -# CHECK-NEXT: 1 4 0.50 zip1 z0.b, z0.b, z0.b -# CHECK-NEXT: 1 4 0.50 zip1 z0.d, z0.d, z0.d -# CHECK-NEXT: 1 4 0.50 zip1 z0.h, z0.h, z0.h -# CHECK-NEXT: 1 4 0.50 zip1 z0.s, z0.s, z0.s -# CHECK-NEXT: 1 4 0.50 zip1 z31.b, z31.b, z31.b -# CHECK-NEXT: 1 4 0.50 zip1 z31.d, z31.d, z31.d -# CHECK-NEXT: 1 4 0.50 zip1 z31.h, z31.h, z31.h -# CHECK-NEXT: 1 4 0.50 zip1 z31.s, z31.s, z31.s -# CHECK-NEXT: 1 6 1.00 zip2 p0.b, p0.b, p0.b -# CHECK-NEXT: 1 6 1.00 zip2 p0.d, p0.d, p0.d -# CHECK-NEXT: 1 6 1.00 zip2 p0.h, p0.h, p0.h -# CHECK-NEXT: 1 6 1.00 zip2 p0.s, p0.s, p0.s -# CHECK-NEXT: 1 6 1.00 zip2 p15.b, p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 zip2 p15.d, p15.d, p15.d -# CHECK-NEXT: 1 6 1.00 zip2 p15.h, p15.h, p15.h -# CHECK-NEXT: 1 6 1.00 zip2 p15.s, p15.s, p15.s -# CHECK-NEXT: 1 4 0.50 zip2 z0.b, z0.b, z0.b -# CHECK-NEXT: 1 4 0.50 zip2 z0.d, z0.d, z0.d -# CHECK-NEXT: 1 4 0.50 zip2 z0.h, z0.h, z0.h -# CHECK-NEXT: 1 4 0.50 zip2 z0.s, z0.s, z0.s -# CHECK-NEXT: 1 4 0.50 zip2 z31.b, z31.b, z31.b -# CHECK-NEXT: 1 4 0.50 zip2 z31.d, z31.d, z31.d -# CHECK-NEXT: 1 4 0.50 zip2 z31.h, z31.h, z31.h -# CHECK-NEXT: 1 4 0.50 zip2 z31.s, z31.s, z31.s +# CHECK-NEXT: 1 4 0.50 xar z0.b, z0.b, z1.b, #1 +# CHECK-NEXT: 1 4 0.50 xar z0.d, z0.d, z1.d, #1 +# CHECK-NEXT: 1 4 0.50 xar z0.h, z0.h, z1.h, #1 +# CHECK-NEXT: 1 4 0.50 xar z0.s, z0.s, z1.s, #1 +# CHECK-NEXT: 1 4 0.50 xar z31.b, z31.b, z30.b, #8 +# CHECK-NEXT: 1 4 0.50 xar z31.d, z31.d, z30.d, #64 +# CHECK-NEXT: 1 4 0.50 xar z31.h, z31.h, z30.h, #16 +# CHECK-NEXT: 1 4 0.50 xar z31.s, z31.s, z30.s, #32 +# CHECK-NEXT: 1 2 1.00 zip1 p0.b, p0.b, p0.b +# CHECK-NEXT: 1 2 1.00 zip1 p0.d, p0.d, p0.d +# CHECK-NEXT: 1 2 1.00 zip1 p0.h, p0.h, p0.h +# CHECK-NEXT: 1 2 1.00 zip1 p0.s, p0.s, p0.s +# CHECK-NEXT: 1 2 1.00 zip1 p15.b, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 zip1 p15.d, p15.d, p15.d +# CHECK-NEXT: 1 2 1.00 zip1 p15.h, p15.h, p15.h +# CHECK-NEXT: 1 2 1.00 zip1 p15.s, p15.s, p15.s +# CHECK-NEXT: 1 3 0.50 zip1 z0.b, z0.b, z0.b +# CHECK-NEXT: 1 3 0.50 zip1 z0.d, z0.d, z0.d +# CHECK-NEXT: 1 3 0.50 zip1 z0.h, z0.h, z0.h +# CHECK-NEXT: 1 3 0.50 zip1 z0.s, z0.s, z0.s +# CHECK-NEXT: 1 3 0.50 zip1 z31.b, z31.b, z31.b +# CHECK-NEXT: 1 3 0.50 zip1 z31.d, z31.d, z31.d +# CHECK-NEXT: 1 3 0.50 zip1 z31.h, z31.h, z31.h +# CHECK-NEXT: 1 3 0.50 zip1 z31.s, z31.s, z31.s +# CHECK-NEXT: 1 2 1.00 zip2 p0.b, p0.b, p0.b +# CHECK-NEXT: 1 2 1.00 zip2 p0.d, p0.d, p0.d +# CHECK-NEXT: 1 2 1.00 zip2 p0.h, p0.h, p0.h +# CHECK-NEXT: 1 2 1.00 zip2 p0.s, p0.s, p0.s +# CHECK-NEXT: 1 2 1.00 zip2 p15.b, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 zip2 p15.d, p15.d, p15.d +# CHECK-NEXT: 1 2 1.00 zip2 p15.h, p15.h, p15.h +# CHECK-NEXT: 1 2 1.00 zip2 p15.s, p15.s, p15.s +# CHECK-NEXT: 1 3 0.50 zip2 z0.b, z0.b, z0.b +# CHECK-NEXT: 1 3 0.50 zip2 z0.d, z0.d, z0.d +# CHECK-NEXT: 1 3 0.50 zip2 z0.h, z0.h, z0.h +# CHECK-NEXT: 1 3 0.50 zip2 z0.s, z0.s, z0.s +# CHECK-NEXT: 1 3 0.50 zip2 z31.b, z31.b, z31.b +# CHECK-NEXT: 1 3 0.50 zip2 z31.d, z31.d, z31.d +# CHECK-NEXT: 1 3 0.50 zip2 z31.h, z31.h, z31.h +# CHECK-NEXT: 1 3 0.50 zip2 z31.s, z31.s, z31.s # CHECK: Resources: # CHECK-NEXT: [0] - CortexA510UnitALU0 @@ -6848,7 +6848,7 @@ zip2 z31.s, z31.s, z31.s # CHECK: Resource pressure per iteration: # CHECK-NEXT: [0] [1.0] [1.1] [2] [3] [4] [5] [6] [7] [8] [9] [10.0] [10.1] [11] -# CHECK-NEXT: 79.00 75.00 75.00 9.00 - 240.00 3698.00 - - 1290.00 924.00 199.50 199.50 670.00 +# CHECK-NEXT: 79.00 75.00 75.00 9.00 - 209.00 3667.00 - - 1290.00 924.00 199.50 199.50 670.00 # CHECK: Resource pressure by instruction: # CHECK-NEXT: [0] [1.0] [1.1] [2] [3] [4] [5] [6] [7] [8] [9] [10.0] [10.1] [11] Instructions: @@ -7844,8 +7844,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1b { z0.d }, p0/z, [x0] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ld1b { z0.d }, p0/z, [z0.d] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1b { z0.h }, p0/z, [x0] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ld1b { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ld1b { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ld1b { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ld1b { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1b { z0.s }, p0/z, [x0] # CHECK-NEXT: - - - - - - 9.00 - - - - - - - ld1b { z0.s }, p0/z, [z0.s] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1b { z21.b }, p5/z, [x10, #5, mul vl] @@ -7882,8 +7882,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1h { z0.d }, p0/z, [x0] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ld1h { z0.d }, p0/z, [z0.d] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1h { z0.h }, p0/z, [x0] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ld1h { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ld1h { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ld1h { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ld1h { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1h { z0.s }, p0/z, [x0] # CHECK-NEXT: - - - - - - 9.00 - - - - - - - ld1h { z0.s }, p0/z, [z0.s] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1h { z21.d }, p5/z, [x10, #5, mul vl] @@ -7899,8 +7899,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ld1h { z31.d }, p7/z, [z31.d, #62] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1h { z31.h }, p7/z, [sp, #-1, mul vl] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1h { z31.s }, p7/z, [sp, #-1, mul vl] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ld1h { z31.s }, p7/z, [sp, z31.s, sxtw #1] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ld1h { z31.s }, p7/z, [sp, z31.s, uxtw #1] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ld1h { z31.s }, p7/z, [sp, z31.s, sxtw #1] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ld1h { z31.s }, p7/z, [sp, z31.s, uxtw #1] # CHECK-NEXT: - - - - - - 9.00 - - - - - - - ld1h { z31.s }, p7/z, [z31.s, #62] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1h { z5.h }, p3/z, [sp, x16, lsl #1] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1h { z5.h }, p3/z, [x17, x16, lsl #1] @@ -7961,7 +7961,7 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1sb { z0.h }, p0/z, [sp, x0] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1sb { z0.h }, p0/z, [x0, x0] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1sb { z0.h }, p0/z, [x0] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ld1sb { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ld1sb { z0.s }, p0/z, [x0, z0.s, sxtw] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1sb { z0.s }, p0/z, [x0] # CHECK-NEXT: - - - - - - 9.00 - - - - - - - ld1sb { z0.s }, p0/z, [z0.s] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1sb { z21.d }, p5/z, [x10, #5, mul vl] @@ -7981,8 +7981,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ld1sh { z0.d }, p0/z, [x0, z0.d, uxtw #1] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1sh { z0.d }, p0/z, [x0] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ld1sh { z0.d }, p0/z, [z0.d] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ld1sh { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ld1sh { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ld1sh { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ld1sh { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1sh { z0.s }, p0/z, [x0] # CHECK-NEXT: - - - - - - 9.00 - - - - - - - ld1sh { z0.s }, p0/z, [z0.s] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1sh { z21.d }, p5/z, [x10, #5, mul vl] @@ -7997,8 +7997,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ld1sh { z31.d }, p7/z, [sp, z31.d] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ld1sh { z31.d }, p7/z, [z31.d, #62] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1sh { z31.s }, p7/z, [sp, #-1, mul vl] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ld1sh { z31.s }, p7/z, [sp, z31.s, sxtw #1] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ld1sh { z31.s }, p7/z, [sp, z31.s, uxtw #1] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ld1sh { z31.s }, p7/z, [sp, z31.s, sxtw #1] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ld1sh { z31.s }, p7/z, [sp, z31.s, uxtw #1] # CHECK-NEXT: - - - - - - 9.00 - - - - - - - ld1sh { z31.s }, p7/z, [z31.s, #62] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ld1sw { z0.d }, p0/z, [x0, z0.d, sxtw #2] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ld1sw { z0.d }, p0/z, [x0, z0.d, uxtw #2] @@ -8017,8 +8017,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ld1w { z0.d }, p0/z, [x0, z0.d, uxtw #2] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1w { z0.d }, p0/z, [x0] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ld1w { z0.d }, p0/z, [z0.d] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ld1w { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ld1w { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ld1w { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ld1w { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1w { z0.s }, p0/z, [x0] # CHECK-NEXT: - - - - - - 9.00 - - - - - - - ld1w { z0.s }, p0/z, [z0.s] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1w { z21.d }, p5/z, [x10, #5, mul vl] @@ -8033,8 +8033,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ld1w { z31.d }, p7/z, [sp, z31.d] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ld1w { z31.d }, p7/z, [z31.d, #124] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1w { z31.s }, p7/z, [sp, #-1, mul vl] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ld1w { z31.s }, p7/z, [sp, z31.s, sxtw #2] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ld1w { z31.s }, p7/z, [sp, z31.s, uxtw #2] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ld1w { z31.s }, p7/z, [sp, z31.s, sxtw #2] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ld1w { z31.s }, p7/z, [sp, z31.s, uxtw #2] # CHECK-NEXT: - - - - - - 9.00 - - - - - - - ld1w { z31.s }, p7/z, [z31.s, #124] # CHECK-NEXT: - - - - - - 2.00 - - - - - - - ld2b { z0.b, z1.b }, p0/z, [x0, x0] # CHECK-NEXT: - - - - - - 1.00 - - - - - - - ld2b { z0.b, z1.b }, p0/z, [x0] @@ -8100,8 +8100,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1b { z0.d }, p0/z, [z0.d] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldff1b { z0.h }, p0/z, [x0, x0] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldff1b { z0.s }, p0/z, [x0, x0] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ldff1b { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ldff1b { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ldff1b { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ldff1b { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: - - - - - - 9.00 - - - - - - - ldff1b { z0.s }, p0/z, [z0.s] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1b { z21.d }, p5/z, [x10, z21.d, sxtw] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1b { z21.d }, p5/z, [x10, z21.d, uxtw] @@ -8128,8 +8128,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1h { z0.d }, p0/z, [z0.d] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldff1h { z0.h }, p0/z, [x0, x0, lsl #1] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldff1h { z0.s }, p0/z, [x0, x0, lsl #1] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ldff1h { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ldff1h { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ldff1h { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ldff1h { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: - - - - - - 9.00 - - - - - - - ldff1h { z0.s }, p0/z, [z0.s] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1h { z21.d }, p5/z, [x10, z21.d, sxtw] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1h { z21.d }, p5/z, [x10, z21.d, uxtw] @@ -8138,16 +8138,16 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldff1h { z31.d }, p7/z, [sp] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1h { z31.d }, p7/z, [z31.d, #62] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldff1h { z31.h }, p7/z, [sp] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ldff1h { z31.s }, p7/z, [sp, z31.s, sxtw #1] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ldff1h { z31.s }, p7/z, [sp, z31.s, uxtw #1] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ldff1h { z31.s }, p7/z, [sp, z31.s, sxtw #1] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ldff1h { z31.s }, p7/z, [sp, z31.s, uxtw #1] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldff1h { z31.s }, p7/z, [sp] # CHECK-NEXT: - - - - - - 9.00 - - - - - - - ldff1h { z31.s }, p7/z, [z31.s, #62] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldff1sb { z0.d }, p0/z, [x0, x0] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1sb { z0.d }, p0/z, [z0.d] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldff1sb { z0.h }, p0/z, [x0, x0] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldff1sb { z0.s }, p0/z, [x0, x0] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ldff1sb { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ldff1sb { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ldff1sb { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ldff1sb { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: - - - - - - 9.00 - - - - - - - ldff1sb { z0.s }, p0/z, [z0.s] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1sb { z21.d }, p5/z, [x10, z21.d, sxtw] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1sb { z21.d }, p5/z, [x10, z21.d, uxtw] @@ -8162,8 +8162,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1sh { z0.d }, p0/z, [x0, z0.d, uxtw #1] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1sh { z0.d }, p0/z, [z0.d] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldff1sh { z0.s }, p0/z, [x0, x0, lsl #1] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ldff1sh { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ldff1sh { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ldff1sh { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ldff1sh { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: - - - - - - 9.00 - - - - - - - ldff1sh { z0.s }, p0/z, [z0.s] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1sh { z21.d }, p5/z, [x10, z21.d, sxtw] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1sh { z21.d }, p5/z, [x10, z21.d, uxtw] @@ -8171,8 +8171,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1sh { z31.d }, p7/z, [sp, z31.d] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldff1sh { z31.d }, p7/z, [sp] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1sh { z31.d }, p7/z, [z31.d, #62] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ldff1sh { z31.s }, p7/z, [sp, z31.s, sxtw #1] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ldff1sh { z31.s }, p7/z, [sp, z31.s, uxtw #1] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ldff1sh { z31.s }, p7/z, [sp, z31.s, sxtw #1] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ldff1sh { z31.s }, p7/z, [sp, z31.s, uxtw #1] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldff1sh { z31.s }, p7/z, [sp] # CHECK-NEXT: - - - - - - 9.00 - - - - - - - ldff1sh { z31.s }, p7/z, [z31.s, #62] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldff1sw { z0.d }, p0/z, [x0, x0, lsl #2] @@ -8190,8 +8190,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1w { z0.d }, p0/z, [x0, z0.d, uxtw #2] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1w { z0.d }, p0/z, [z0.d] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldff1w { z0.s }, p0/z, [x0, x0, lsl #2] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ldff1w { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ldff1w { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ldff1w { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ldff1w { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: - - - - - - 9.00 - - - - - - - ldff1w { z0.s }, p0/z, [z0.s] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1w { z21.d }, p5/z, [x10, z21.d, sxtw] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1w { z21.d }, p5/z, [x10, z21.d, uxtw] @@ -8199,8 +8199,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1w { z31.d }, p7/z, [sp, z31.d] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldff1w { z31.d }, p7/z, [sp] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1w { z31.d }, p7/z, [z31.d, #124] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ldff1w { z31.s }, p7/z, [sp, z31.s, sxtw #2] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ldff1w { z31.s }, p7/z, [sp, z31.s, uxtw #2] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ldff1w { z31.s }, p7/z, [sp, z31.s, sxtw #2] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ldff1w { z31.s }, p7/z, [sp, z31.s, uxtw #2] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldff1w { z31.s }, p7/z, [sp] # CHECK-NEXT: - - - - - - 9.00 - - - - - - - ldff1w { z31.s }, p7/z, [z31.s, #124] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldnf1b { z0.b }, p0/z, [x0] -- GitLab From 4572a2db2c2c48820e79fb65f3810348371db4f1 Mon Sep 17 00:00:00 2001 From: David Green Date: Wed, 17 Apr 2024 19:44:17 +0100 Subject: [PATCH 090/862] [AArch64] Add some test cases for LD2/LD3/LD4 shuffles. NFC --- .../CostModel/AArch64/shuffle-load.ll | 620 ++++++++++++++++++ 1 file changed, 620 insertions(+) diff --git a/llvm/test/Analysis/CostModel/AArch64/shuffle-load.ll b/llvm/test/Analysis/CostModel/AArch64/shuffle-load.ll index ec95a313e11e..106f2f9edc2e 100644 --- a/llvm/test/Analysis/CostModel/AArch64/shuffle-load.ll +++ b/llvm/test/Analysis/CostModel/AArch64/shuffle-load.ll @@ -401,3 +401,623 @@ entry: %lane = shufflevector <2 x i64> %tmp1, <2 x i64> undef, <2 x i32> zeroinitializer ret <2 x i64> %lane } + +define void @vld2(ptr %p) { +; CHECK-LABEL: 'vld2' +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i8 = load <4 x i8>, ptr %p, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i8_0 = shufflevector <4 x i8> %v4i8, <4 x i8> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i8_1 = shufflevector <4 x i8> %v4i8, <4 x i8> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i8 = load <8 x i8>, ptr %p, align 8 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = shufflevector <8 x i8> %v8i8, <8 x i8> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i8_1 = shufflevector <8 x i8> %v8i8, <8 x i8> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i8 = load <16 x i8>, ptr %p, align 16 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i8_0 = shufflevector <16 x i8> %v16i8, <16 x i8> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i8_1 = shufflevector <16 x i8> %v16i8, <16 x i8> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v32i8 = load <32 x i8>, ptr %p, align 32 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v32i8_0 = shufflevector <32 x i8> %v32i8, <32 x i8> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v32i8_1 = shufflevector <32 x i8> %v32i8, <32 x i8> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i16 = load <4 x i16>, ptr %p, align 8 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i16_0 = shufflevector <4 x i16> %v4i16, <4 x i16> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i16_1 = shufflevector <4 x i16> %v4i16, <4 x i16> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i16 = load <8 x i16>, ptr %p, align 16 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i16_0 = shufflevector <8 x i16> %v8i16, <8 x i16> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i16_1 = shufflevector <8 x i16> %v8i16, <8 x i16> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v16i16 = load <16 x i16>, ptr %p, align 32 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i16_0 = shufflevector <16 x i16> %v16i16, <16 x i16> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i16_1 = shufflevector <16 x i16> %v16i16, <16 x i16> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v32i16 = load <32 x i16>, ptr %p, align 64 +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v32i16_0 = shufflevector <32 x i16> %v32i16, <32 x i16> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v32i16_1 = shufflevector <32 x i16> %v32i16, <32 x i16> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i32 = load <4 x i32>, ptr %p, align 16 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i32_0 = shufflevector <4 x i32> %v4i32, <4 x i32> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i32_1 = shufflevector <4 x i32> %v4i32, <4 x i32> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8i32 = load <8 x i32>, ptr %p, align 32 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i32_0 = shufflevector <8 x i32> %v8i32, <8 x i32> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i32_1 = shufflevector <8 x i32> %v8i32, <8 x i32> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v16i32 = load <16 x i32>, ptr %p, align 64 +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v16i32_0 = shufflevector <16 x i32> %v16i32, <16 x i32> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v16i32_1 = shufflevector <16 x i32> %v16i32, <16 x i32> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v32i32 = load <32 x i32>, ptr %p, align 128 +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v32i32_0 = shufflevector <32 x i32> %v32i32, <32 x i32> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v32i32_1 = shufflevector <32 x i32> %v32i32, <32 x i32> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2i64 = load <4 x i64>, ptr %p, align 32 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_0 = shufflevector <4 x i64> %v2i64, <4 x i64> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_1 = shufflevector <4 x i64> %v2i64, <4 x i64> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4i64 = load <8 x i64>, ptr %p, align 64 +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_0 = shufflevector <8 x i64> %v4i64, <8 x i64> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_1 = shufflevector <8 x i64> %v4i64, <8 x i64> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i64 = load <16 x i64>, ptr %p, align 128 +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_0 = shufflevector <16 x i64> %v8i64, <16 x i64> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_1 = shufflevector <16 x i64> %v8i64, <16 x i64> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i64 = load <32 x i64>, ptr %p, align 256 +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_0 = shufflevector <32 x i64> %v16i64, <32 x i64> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_1 = shufflevector <32 x i64> %v16i64, <32 x i64> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; CODESIZE-LABEL: 'vld2' +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i8 = load <4 x i8>, ptr %p, align 4 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i8_0 = shufflevector <4 x i8> %v4i8, <4 x i8> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i8_1 = shufflevector <4 x i8> %v4i8, <4 x i8> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i8 = load <8 x i8>, ptr %p, align 8 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = shufflevector <8 x i8> %v8i8, <8 x i8> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i8_1 = shufflevector <8 x i8> %v8i8, <8 x i8> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i8 = load <16 x i8>, ptr %p, align 16 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i8_0 = shufflevector <16 x i8> %v16i8, <16 x i8> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i8_1 = shufflevector <16 x i8> %v16i8, <16 x i8> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v32i8 = load <32 x i8>, ptr %p, align 32 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v32i8_0 = shufflevector <32 x i8> %v32i8, <32 x i8> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v32i8_1 = shufflevector <32 x i8> %v32i8, <32 x i8> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i16 = load <4 x i16>, ptr %p, align 8 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i16_0 = shufflevector <4 x i16> %v4i16, <4 x i16> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i16_1 = shufflevector <4 x i16> %v4i16, <4 x i16> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i16 = load <8 x i16>, ptr %p, align 16 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i16_0 = shufflevector <8 x i16> %v8i16, <8 x i16> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i16_1 = shufflevector <8 x i16> %v8i16, <8 x i16> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v16i16 = load <16 x i16>, ptr %p, align 32 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i16_0 = shufflevector <16 x i16> %v16i16, <16 x i16> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i16_1 = shufflevector <16 x i16> %v16i16, <16 x i16> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v32i16 = load <32 x i16>, ptr %p, align 64 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v32i16_0 = shufflevector <32 x i16> %v32i16, <32 x i16> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v32i16_1 = shufflevector <32 x i16> %v32i16, <32 x i16> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i32 = load <4 x i32>, ptr %p, align 16 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i32_0 = shufflevector <4 x i32> %v4i32, <4 x i32> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i32_1 = shufflevector <4 x i32> %v4i32, <4 x i32> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8i32 = load <8 x i32>, ptr %p, align 32 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i32_0 = shufflevector <8 x i32> %v8i32, <8 x i32> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i32_1 = shufflevector <8 x i32> %v8i32, <8 x i32> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v16i32 = load <16 x i32>, ptr %p, align 64 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v16i32_0 = shufflevector <16 x i32> %v16i32, <16 x i32> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v16i32_1 = shufflevector <16 x i32> %v16i32, <16 x i32> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v32i32 = load <32 x i32>, ptr %p, align 128 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v32i32_0 = shufflevector <32 x i32> %v32i32, <32 x i32> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v32i32_1 = shufflevector <32 x i32> %v32i32, <32 x i32> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2i64 = load <4 x i64>, ptr %p, align 32 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_0 = shufflevector <4 x i64> %v2i64, <4 x i64> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_1 = shufflevector <4 x i64> %v2i64, <4 x i64> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4i64 = load <8 x i64>, ptr %p, align 64 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_0 = shufflevector <8 x i64> %v4i64, <8 x i64> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_1 = shufflevector <8 x i64> %v4i64, <8 x i64> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i64 = load <16 x i64>, ptr %p, align 128 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_0 = shufflevector <16 x i64> %v8i64, <16 x i64> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_1 = shufflevector <16 x i64> %v8i64, <16 x i64> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i64 = load <32 x i64>, ptr %p, align 256 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_0 = shufflevector <32 x i64> %v16i64, <32 x i64> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_1 = shufflevector <32 x i64> %v16i64, <32 x i64> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; + %v4i8 = load <4 x i8>, ptr %p + %v4i8_0 = shufflevector <4 x i8> %v4i8, <4 x i8> undef, <2 x i32> + %v4i8_1 = shufflevector <4 x i8> %v4i8, <4 x i8> undef, <2 x i32> + %v8i8 = load <8 x i8>, ptr %p + %v8i8_0 = shufflevector <8 x i8> %v8i8, <8 x i8> undef, <4 x i32> + %v8i8_1 = shufflevector <8 x i8> %v8i8, <8 x i8> undef, <4 x i32> + %v16i8 = load <16 x i8>, ptr %p + %v16i8_0 = shufflevector <16 x i8> %v16i8, <16 x i8> undef, <8 x i32> + %v16i8_1 = shufflevector <16 x i8> %v16i8, <16 x i8> undef, <8 x i32> + %v32i8 = load <32 x i8>, ptr %p + %v32i8_0 = shufflevector <32 x i8> %v32i8, <32 x i8> undef, <16 x i32> + %v32i8_1 = shufflevector <32 x i8> %v32i8, <32 x i8> undef, <16 x i32> + + %v4i16 = load <4 x i16>, ptr %p + %v4i16_0 = shufflevector <4 x i16> %v4i16, <4 x i16> undef, <2 x i32> + %v4i16_1 = shufflevector <4 x i16> %v4i16, <4 x i16> undef, <2 x i32> + %v8i16 = load <8 x i16>, ptr %p + %v8i16_0 = shufflevector <8 x i16> %v8i16, <8 x i16> undef, <4 x i32> + %v8i16_1 = shufflevector <8 x i16> %v8i16, <8 x i16> undef, <4 x i32> + %v16i16 = load <16 x i16>, ptr %p + %v16i16_0 = shufflevector <16 x i16> %v16i16, <16 x i16> undef, <8 x i32> + %v16i16_1 = shufflevector <16 x i16> %v16i16, <16 x i16> undef, <8 x i32> + %v32i16 = load <32 x i16>, ptr %p + %v32i16_0 = shufflevector <32 x i16> %v32i16, <32 x i16> undef, <16 x i32> + %v32i16_1 = shufflevector <32 x i16> %v32i16, <32 x i16> undef, <16 x i32> + + %v4i32 = load <4 x i32>, ptr %p + %v4i32_0 = shufflevector <4 x i32> %v4i32, <4 x i32> undef, <2 x i32> + %v4i32_1 = shufflevector <4 x i32> %v4i32, <4 x i32> undef, <2 x i32> + %v8i32 = load <8 x i32>, ptr %p + %v8i32_0 = shufflevector <8 x i32> %v8i32, <8 x i32> undef, <4 x i32> + %v8i32_1 = shufflevector <8 x i32> %v8i32, <8 x i32> undef, <4 x i32> + %v16i32 = load <16 x i32>, ptr %p + %v16i32_0 = shufflevector <16 x i32> %v16i32, <16 x i32> undef, <8 x i32> + %v16i32_1 = shufflevector <16 x i32> %v16i32, <16 x i32> undef, <8 x i32> + %v32i32 = load <32 x i32>, ptr %p + %v32i32_0 = shufflevector <32 x i32> %v32i32, <32 x i32> undef, <16 x i32> + %v32i32_1 = shufflevector <32 x i32> %v32i32, <32 x i32> undef, <16 x i32> + + %v2i64 = load <4 x i64>, ptr %p + %v2i64_0 = shufflevector <4 x i64> %v2i64, <4 x i64> undef, <2 x i32> + %v2i64_1 = shufflevector <4 x i64> %v2i64, <4 x i64> undef, <2 x i32> + %v4i64 = load <8 x i64>, ptr %p + %v4i64_0 = shufflevector <8 x i64> %v4i64, <8 x i64> undef, <4 x i32> + %v4i64_1 = shufflevector <8 x i64> %v4i64, <8 x i64> undef, <4 x i32> + %v8i64 = load <16 x i64>, ptr %p + %v8i64_0 = shufflevector <16 x i64> %v8i64, <16 x i64> undef, <8 x i32> + %v8i64_1 = shufflevector <16 x i64> %v8i64, <16 x i64> undef, <8 x i32> + %v16i64 = load <32 x i64>, ptr %p + %v16i64_0 = shufflevector <32 x i64> %v16i64, <32 x i64> undef, <16 x i32> + %v16i64_1 = shufflevector <32 x i64> %v16i64, <32 x i64> undef, <16 x i32> + + ret void +} + + +define void @vld3(ptr %p) { +; CHECK-LABEL: 'vld3' +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i8 = load <6 x i8>, ptr %p, align 8 +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i8_0 = shufflevector <6 x i8> %v2i8, <6 x i8> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i8_1 = shufflevector <6 x i8> %v2i8, <6 x i8> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i8_2 = shufflevector <6 x i8> %v2i8, <6 x i8> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i8 = load <12 x i8>, ptr %p, align 16 +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i8_0 = shufflevector <12 x i8> %v4i8, <12 x i8> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i8_1 = shufflevector <12 x i8> %v4i8, <12 x i8> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i8_2 = shufflevector <12 x i8> %v4i8, <12 x i8> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8i8 = load <24 x i8>, ptr %p, align 32 +; CHECK-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %v8i8_0 = shufflevector <24 x i8> %v8i8, <24 x i8> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %v8i8_1 = shufflevector <24 x i8> %v8i8, <24 x i8> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %v8i8_2 = shufflevector <24 x i8> %v8i8, <24 x i8> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v16i8 = load <48 x i8>, ptr %p, align 64 +; CHECK-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v16i8_0 = shufflevector <48 x i8> %v16i8, <48 x i8> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i8_1 = shufflevector <48 x i8> %v16i8, <48 x i8> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v16i8_2 = shufflevector <48 x i8> %v16i8, <48 x i8> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i16 = load <6 x i16>, ptr %p, align 16 +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i16_0 = shufflevector <6 x i16> %v2i16, <6 x i16> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i16_1 = shufflevector <6 x i16> %v2i16, <6 x i16> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i16_2 = shufflevector <6 x i16> %v2i16, <6 x i16> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i16 = load <12 x i16>, ptr %p, align 32 +; CHECK-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %v4i16_0 = shufflevector <12 x i16> %v4i16, <12 x i16> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %v4i16_1 = shufflevector <12 x i16> %v4i16, <12 x i16> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %v4i16_2 = shufflevector <12 x i16> %v4i16, <12 x i16> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i16 = load <24 x i16>, ptr %p, align 64 +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v8i16_0 = shufflevector <24 x i16> %v8i16, <24 x i16> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i16_1 = shufflevector <24 x i16> %v8i16, <24 x i16> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v8i16_2 = shufflevector <24 x i16> %v8i16, <24 x i16> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i16 = load <48 x i16>, ptr %p, align 128 +; CHECK-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %v16i16_0 = shufflevector <48 x i16> %v16i16, <48 x i16> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i16_1 = shufflevector <48 x i16> %v16i16, <48 x i16> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %v16i16_2 = shufflevector <48 x i16> %v16i16, <48 x i16> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2i32 = load <6 x i32>, ptr %p, align 32 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i32_0 = shufflevector <6 x i32> %v2i32, <6 x i32> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2i32_1 = shufflevector <6 x i32> %v2i32, <6 x i32> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i32_2 = shufflevector <6 x i32> %v2i32, <6 x i32> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4i32 = load <12 x i32>, ptr %p, align 64 +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_0 = shufflevector <12 x i32> %v4i32, <12 x i32> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4i32_1 = shufflevector <12 x i32> %v4i32, <12 x i32> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_2 = shufflevector <12 x i32> %v4i32, <12 x i32> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i32 = load <24 x i32>, ptr %p, align 128 +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8i32_0 = shufflevector <24 x i32> %v8i32, <24 x i32> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i32_1 = shufflevector <24 x i32> %v8i32, <24 x i32> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8i32_2 = shufflevector <24 x i32> %v8i32, <24 x i32> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i32 = load <48 x i32>, ptr %p, align 256 +; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v16i32_0 = shufflevector <48 x i32> %v16i32, <48 x i32> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i32_1 = shufflevector <48 x i32> %v16i32, <48 x i32> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v16i32_2 = shufflevector <48 x i32> %v16i32, <48 x i32> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v2i64 = load <6 x i64>, ptr %p, align 64 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_0 = shufflevector <6 x i64> %v2i64, <6 x i64> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_1 = shufflevector <6 x i64> %v2i64, <6 x i64> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_2 = shufflevector <6 x i64> %v2i64, <6 x i64> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i64 = load <12 x i64>, ptr %p, align 128 +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_0 = shufflevector <12 x i64> %v4i64, <12 x i64> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_1 = shufflevector <12 x i64> %v4i64, <12 x i64> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_2 = shufflevector <12 x i64> %v4i64, <12 x i64> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v8i64 = load <24 x i64>, ptr %p, align 256 +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_0 = shufflevector <24 x i64> %v8i64, <24 x i64> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_1 = shufflevector <24 x i64> %v8i64, <24 x i64> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_2 = shufflevector <24 x i64> %v8i64, <24 x i64> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v16i64 = load <48 x i64>, ptr %p, align 512 +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_0 = shufflevector <48 x i64> %v16i64, <48 x i64> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_1 = shufflevector <48 x i64> %v16i64, <48 x i64> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_2 = shufflevector <48 x i64> %v16i64, <48 x i64> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; CODESIZE-LABEL: 'vld3' +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i8 = load <6 x i8>, ptr %p, align 8 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i8_0 = shufflevector <6 x i8> %v2i8, <6 x i8> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i8_1 = shufflevector <6 x i8> %v2i8, <6 x i8> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i8_2 = shufflevector <6 x i8> %v2i8, <6 x i8> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i8 = load <12 x i8>, ptr %p, align 16 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i8_0 = shufflevector <12 x i8> %v4i8, <12 x i8> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i8_1 = shufflevector <12 x i8> %v4i8, <12 x i8> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i8_2 = shufflevector <12 x i8> %v4i8, <12 x i8> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8i8 = load <24 x i8>, ptr %p, align 32 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %v8i8_0 = shufflevector <24 x i8> %v8i8, <24 x i8> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %v8i8_1 = shufflevector <24 x i8> %v8i8, <24 x i8> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %v8i8_2 = shufflevector <24 x i8> %v8i8, <24 x i8> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v16i8 = load <48 x i8>, ptr %p, align 64 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v16i8_0 = shufflevector <48 x i8> %v16i8, <48 x i8> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i8_1 = shufflevector <48 x i8> %v16i8, <48 x i8> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v16i8_2 = shufflevector <48 x i8> %v16i8, <48 x i8> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i16 = load <6 x i16>, ptr %p, align 16 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i16_0 = shufflevector <6 x i16> %v2i16, <6 x i16> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i16_1 = shufflevector <6 x i16> %v2i16, <6 x i16> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i16_2 = shufflevector <6 x i16> %v2i16, <6 x i16> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i16 = load <12 x i16>, ptr %p, align 32 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %v4i16_0 = shufflevector <12 x i16> %v4i16, <12 x i16> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %v4i16_1 = shufflevector <12 x i16> %v4i16, <12 x i16> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %v4i16_2 = shufflevector <12 x i16> %v4i16, <12 x i16> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i16 = load <24 x i16>, ptr %p, align 64 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v8i16_0 = shufflevector <24 x i16> %v8i16, <24 x i16> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i16_1 = shufflevector <24 x i16> %v8i16, <24 x i16> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v8i16_2 = shufflevector <24 x i16> %v8i16, <24 x i16> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i16 = load <48 x i16>, ptr %p, align 128 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %v16i16_0 = shufflevector <48 x i16> %v16i16, <48 x i16> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i16_1 = shufflevector <48 x i16> %v16i16, <48 x i16> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %v16i16_2 = shufflevector <48 x i16> %v16i16, <48 x i16> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2i32 = load <6 x i32>, ptr %p, align 32 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i32_0 = shufflevector <6 x i32> %v2i32, <6 x i32> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2i32_1 = shufflevector <6 x i32> %v2i32, <6 x i32> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i32_2 = shufflevector <6 x i32> %v2i32, <6 x i32> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4i32 = load <12 x i32>, ptr %p, align 64 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_0 = shufflevector <12 x i32> %v4i32, <12 x i32> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4i32_1 = shufflevector <12 x i32> %v4i32, <12 x i32> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_2 = shufflevector <12 x i32> %v4i32, <12 x i32> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i32 = load <24 x i32>, ptr %p, align 128 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8i32_0 = shufflevector <24 x i32> %v8i32, <24 x i32> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i32_1 = shufflevector <24 x i32> %v8i32, <24 x i32> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8i32_2 = shufflevector <24 x i32> %v8i32, <24 x i32> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i32 = load <48 x i32>, ptr %p, align 256 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v16i32_0 = shufflevector <48 x i32> %v16i32, <48 x i32> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i32_1 = shufflevector <48 x i32> %v16i32, <48 x i32> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v16i32_2 = shufflevector <48 x i32> %v16i32, <48 x i32> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v2i64 = load <6 x i64>, ptr %p, align 64 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_0 = shufflevector <6 x i64> %v2i64, <6 x i64> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_1 = shufflevector <6 x i64> %v2i64, <6 x i64> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_2 = shufflevector <6 x i64> %v2i64, <6 x i64> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i64 = load <12 x i64>, ptr %p, align 128 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_0 = shufflevector <12 x i64> %v4i64, <12 x i64> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_1 = shufflevector <12 x i64> %v4i64, <12 x i64> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_2 = shufflevector <12 x i64> %v4i64, <12 x i64> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v8i64 = load <24 x i64>, ptr %p, align 256 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_0 = shufflevector <24 x i64> %v8i64, <24 x i64> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_1 = shufflevector <24 x i64> %v8i64, <24 x i64> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_2 = shufflevector <24 x i64> %v8i64, <24 x i64> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v16i64 = load <48 x i64>, ptr %p, align 512 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_0 = shufflevector <48 x i64> %v16i64, <48 x i64> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_1 = shufflevector <48 x i64> %v16i64, <48 x i64> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_2 = shufflevector <48 x i64> %v16i64, <48 x i64> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; + %v2i8 = load <6 x i8>, ptr %p + %v2i8_0 = shufflevector <6 x i8> %v2i8, <6 x i8> undef, <2 x i32> + %v2i8_1 = shufflevector <6 x i8> %v2i8, <6 x i8> undef, <2 x i32> + %v2i8_2 = shufflevector <6 x i8> %v2i8, <6 x i8> undef, <2 x i32> + %v4i8 = load <12 x i8>, ptr %p + %v4i8_0 = shufflevector <12 x i8> %v4i8, <12 x i8> undef, <4 x i32> + %v4i8_1 = shufflevector <12 x i8> %v4i8, <12 x i8> undef, <4 x i32> + %v4i8_2 = shufflevector <12 x i8> %v4i8, <12 x i8> undef, <4 x i32> + %v8i8 = load <24 x i8>, ptr %p + %v8i8_0 = shufflevector <24 x i8> %v8i8, <24 x i8> undef, <8 x i32> + %v8i8_1 = shufflevector <24 x i8> %v8i8, <24 x i8> undef, <8 x i32> + %v8i8_2 = shufflevector <24 x i8> %v8i8, <24 x i8> undef, <8 x i32> + %v16i8 = load <48 x i8>, ptr %p + %v16i8_0 = shufflevector <48 x i8> %v16i8, <48 x i8> undef, <16 x i32> + %v16i8_1 = shufflevector <48 x i8> %v16i8, <48 x i8> undef, <16 x i32> + %v16i8_2 = shufflevector <48 x i8> %v16i8, <48 x i8> undef, <16 x i32> + + %v2i16 = load <6 x i16>, ptr %p + %v2i16_0 = shufflevector <6 x i16> %v2i16, <6 x i16> undef, <2 x i32> + %v2i16_1 = shufflevector <6 x i16> %v2i16, <6 x i16> undef, <2 x i32> + %v2i16_2 = shufflevector <6 x i16> %v2i16, <6 x i16> undef, <2 x i32> + %v4i16 = load <12 x i16>, ptr %p + %v4i16_0 = shufflevector <12 x i16> %v4i16, <12 x i16> undef, <4 x i32> + %v4i16_1 = shufflevector <12 x i16> %v4i16, <12 x i16> undef, <4 x i32> + %v4i16_2 = shufflevector <12 x i16> %v4i16, <12 x i16> undef, <4 x i32> + %v8i16 = load <24 x i16>, ptr %p + %v8i16_0 = shufflevector <24 x i16> %v8i16, <24 x i16> undef, <8 x i32> + %v8i16_1 = shufflevector <24 x i16> %v8i16, <24 x i16> undef, <8 x i32> + %v8i16_2 = shufflevector <24 x i16> %v8i16, <24 x i16> undef, <8 x i32> + %v16i16 = load <48 x i16>, ptr %p + %v16i16_0 = shufflevector <48 x i16> %v16i16, <48 x i16> undef, <16 x i32> + %v16i16_1 = shufflevector <48 x i16> %v16i16, <48 x i16> undef, <16 x i32> + %v16i16_2 = shufflevector <48 x i16> %v16i16, <48 x i16> undef, <16 x i32> + + %v2i32 = load <6 x i32>, ptr %p + %v2i32_0 = shufflevector <6 x i32> %v2i32, <6 x i32> undef, <2 x i32> + %v2i32_1 = shufflevector <6 x i32> %v2i32, <6 x i32> undef, <2 x i32> + %v2i32_2 = shufflevector <6 x i32> %v2i32, <6 x i32> undef, <2 x i32> + %v4i32 = load <12 x i32>, ptr %p + %v4i32_0 = shufflevector <12 x i32> %v4i32, <12 x i32> undef, <4 x i32> + %v4i32_1 = shufflevector <12 x i32> %v4i32, <12 x i32> undef, <4 x i32> + %v4i32_2 = shufflevector <12 x i32> %v4i32, <12 x i32> undef, <4 x i32> + %v8i32 = load <24 x i32>, ptr %p + %v8i32_0 = shufflevector <24 x i32> %v8i32, <24 x i32> undef, <8 x i32> + %v8i32_1 = shufflevector <24 x i32> %v8i32, <24 x i32> undef, <8 x i32> + %v8i32_2 = shufflevector <24 x i32> %v8i32, <24 x i32> undef, <8 x i32> + %v16i32 = load <48 x i32>, ptr %p + %v16i32_0 = shufflevector <48 x i32> %v16i32, <48 x i32> undef, <16 x i32> + %v16i32_1 = shufflevector <48 x i32> %v16i32, <48 x i32> undef, <16 x i32> + %v16i32_2 = shufflevector <48 x i32> %v16i32, <48 x i32> undef, <16 x i32> + + %v2i64 = load <6 x i64>, ptr %p + %v2i64_0 = shufflevector <6 x i64> %v2i64, <6 x i64> undef, <2 x i32> + %v2i64_1 = shufflevector <6 x i64> %v2i64, <6 x i64> undef, <2 x i32> + %v2i64_2 = shufflevector <6 x i64> %v2i64, <6 x i64> undef, <2 x i32> + %v4i64 = load <12 x i64>, ptr %p + %v4i64_0 = shufflevector <12 x i64> %v4i64, <12 x i64> undef, <4 x i32> + %v4i64_1 = shufflevector <12 x i64> %v4i64, <12 x i64> undef, <4 x i32> + %v4i64_2 = shufflevector <12 x i64> %v4i64, <12 x i64> undef, <4 x i32> + %v8i64 = load <24 x i64>, ptr %p + %v8i64_0 = shufflevector <24 x i64> %v8i64, <24 x i64> undef, <8 x i32> + %v8i64_1 = shufflevector <24 x i64> %v8i64, <24 x i64> undef, <8 x i32> + %v8i64_2 = shufflevector <24 x i64> %v8i64, <24 x i64> undef, <8 x i32> + %v16i64 = load <48 x i64>, ptr %p + %v16i64_0 = shufflevector <48 x i64> %v16i64, <48 x i64> undef, <16 x i32> + %v16i64_1 = shufflevector <48 x i64> %v16i64, <48 x i64> undef, <16 x i32> + %v16i64_2 = shufflevector <48 x i64> %v16i64, <48 x i64> undef, <16 x i32> + + ret void +} + +define void @vld4(ptr %p) { +; CHECK-LABEL: 'vld4' +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i8 = load <8 x i8>, ptr %p, align 8 +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i8_0 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i8_1 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i8_2 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i8_3 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i8 = load <16 x i8>, ptr %p, align 16 +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i8_0 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i8_1 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i8_2 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i8_3 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8i8 = load <32 x i8>, ptr %p, align 32 +; CHECK-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %v8i8_0 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %v8i8_1 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %v8i8_2 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %v8i8_3 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v16i8 = load <64 x i8>, ptr %p, align 64 +; CHECK-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v16i8_0 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v16i8_1 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v16i8_2 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v16i8_3 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i16 = load <8 x i16>, ptr %p, align 16 +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i16_0 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i16_1 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i16_2 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i16_3 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i16 = load <16 x i16>, ptr %p, align 32 +; CHECK-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %v4i16_0 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %v4i16_1 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %v4i16_2 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %v4i16_3 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i16 = load <32 x i16>, ptr %p, align 64 +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v8i16_0 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v8i16_1 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v8i16_2 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v8i16_3 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i16 = load <64 x i16>, ptr %p, align 128 +; CHECK-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %v16i16_0 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %v16i16_1 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %v16i16_2 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %v16i16_3 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2i32 = load <8 x i32>, ptr %p, align 32 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i32_0 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i32_1 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i32_2 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2i32_3 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4i32 = load <16 x i32>, ptr %p, align 64 +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_0 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_1 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_2 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_3 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i32 = load <32 x i32>, ptr %p, align 128 +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8i32_0 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8i32_1 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8i32_2 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8i32_3 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i32 = load <64 x i32>, ptr %p, align 256 +; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v16i32_0 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v16i32_1 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v16i32_2 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v16i32_3 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v2i64 = load <8 x i64>, ptr %p, align 64 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_0 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_1 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_2 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_3 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i64 = load <16 x i64>, ptr %p, align 128 +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_0 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_1 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_2 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_3 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v8i64 = load <32 x i64>, ptr %p, align 256 +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_0 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_1 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_2 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_3 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v16i64 = load <64 x i64>, ptr %p, align 512 +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_0 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_1 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_2 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_3 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; CODESIZE-LABEL: 'vld4' +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i8 = load <8 x i8>, ptr %p, align 8 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i8_0 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i8_1 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i8_2 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i8_3 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i8 = load <16 x i8>, ptr %p, align 16 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i8_0 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i8_1 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i8_2 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i8_3 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8i8 = load <32 x i8>, ptr %p, align 32 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %v8i8_0 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %v8i8_1 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %v8i8_2 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %v8i8_3 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v16i8 = load <64 x i8>, ptr %p, align 64 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v16i8_0 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v16i8_1 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v16i8_2 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v16i8_3 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i16 = load <8 x i16>, ptr %p, align 16 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i16_0 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i16_1 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i16_2 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i16_3 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i16 = load <16 x i16>, ptr %p, align 32 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %v4i16_0 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %v4i16_1 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %v4i16_2 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %v4i16_3 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i16 = load <32 x i16>, ptr %p, align 64 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v8i16_0 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v8i16_1 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v8i16_2 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v8i16_3 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i16 = load <64 x i16>, ptr %p, align 128 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %v16i16_0 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %v16i16_1 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %v16i16_2 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %v16i16_3 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2i32 = load <8 x i32>, ptr %p, align 32 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i32_0 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i32_1 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i32_2 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2i32_3 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4i32 = load <16 x i32>, ptr %p, align 64 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_0 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_1 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_2 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_3 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i32 = load <32 x i32>, ptr %p, align 128 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8i32_0 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8i32_1 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8i32_2 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8i32_3 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i32 = load <64 x i32>, ptr %p, align 256 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v16i32_0 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v16i32_1 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v16i32_2 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v16i32_3 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v2i64 = load <8 x i64>, ptr %p, align 64 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_0 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_1 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_2 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_3 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i64 = load <16 x i64>, ptr %p, align 128 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_0 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_1 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_2 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_3 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v8i64 = load <32 x i64>, ptr %p, align 256 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_0 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_1 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_2 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_3 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v16i64 = load <64 x i64>, ptr %p, align 512 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_0 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_1 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_2 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_3 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; + %v2i8 = load <8 x i8>, ptr %p + %v2i8_0 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> + %v2i8_1 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> + %v2i8_2 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> + %v2i8_3 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> + %v4i8 = load <16 x i8>, ptr %p + %v4i8_0 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> + %v4i8_1 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> + %v4i8_2 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> + %v4i8_3 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> + %v8i8 = load <32 x i8>, ptr %p + %v8i8_0 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> + %v8i8_1 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> + %v8i8_2 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> + %v8i8_3 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> + %v16i8 = load <64 x i8>, ptr %p + %v16i8_0 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> + %v16i8_1 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> + %v16i8_2 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> + %v16i8_3 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> + + %v2i16 = load <8 x i16>, ptr %p + %v2i16_0 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> + %v2i16_1 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> + %v2i16_2 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> + %v2i16_3 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> + %v4i16 = load <16 x i16>, ptr %p + %v4i16_0 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> + %v4i16_1 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> + %v4i16_2 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> + %v4i16_3 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> + %v8i16 = load <32 x i16>, ptr %p + %v8i16_0 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> + %v8i16_1 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> + %v8i16_2 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> + %v8i16_3 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> + %v16i16 = load <64 x i16>, ptr %p + %v16i16_0 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> + %v16i16_1 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> + %v16i16_2 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> + %v16i16_3 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> + + %v2i32 = load <8 x i32>, ptr %p + %v2i32_0 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> + %v2i32_1 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> + %v2i32_2 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> + %v2i32_3 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> + %v4i32 = load <16 x i32>, ptr %p + %v4i32_0 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> + %v4i32_1 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> + %v4i32_2 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> + %v4i32_3 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> + %v8i32 = load <32 x i32>, ptr %p + %v8i32_0 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> + %v8i32_1 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> + %v8i32_2 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> + %v8i32_3 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> + %v16i32 = load <64 x i32>, ptr %p + %v16i32_0 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> + %v16i32_1 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> + %v16i32_2 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> + %v16i32_3 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> + + %v2i64 = load <8 x i64>, ptr %p + %v2i64_0 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> + %v2i64_1 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> + %v2i64_2 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> + %v2i64_3 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> + %v4i64 = load <16 x i64>, ptr %p + %v4i64_0 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> + %v4i64_1 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> + %v4i64_2 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> + %v4i64_3 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> + %v8i64 = load <32 x i64>, ptr %p + %v8i64_0 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> + %v8i64_1 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> + %v8i64_2 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> + %v8i64_3 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> + %v16i64 = load <64 x i64>, ptr %p + %v16i64_0 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> + %v16i64_1 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> + %v16i64_2 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> + %v16i64_3 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> + + ret void +} + -- GitLab From 5a0942cd7423069e78fdfb9743a13aedfa7bdee0 Mon Sep 17 00:00:00 2001 From: Youngsuk Kim Date: Wed, 17 Apr 2024 14:45:33 -0400 Subject: [PATCH 091/862] [llvm][NVPTX] Don't emit unused var 'temp_param_reg' (NFC) (#89004) Don't emit unused variable 'temp_param_reg' which has been around since ae556d3ef72dfe5f40a337b7071f42b7bf5b66a4 . --- llvm/lib/Target/NVPTX/NVPTXInstrInfo.td | 3 +-- llvm/test/CodeGen/NVPTX/dynamic_stackalloc.ll | 2 -- llvm/test/CodeGen/NVPTX/i8x4-instructions.ll | 3 --- llvm/test/DebugInfo/NVPTX/dbg-declare-alloca.ll | 1 - .../update_llc_test_checks/Inputs/nvptx-basic.ll.expected | 1 - 5 files changed, 1 insertion(+), 9 deletions(-) diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td index 4daf47cfd4df..cd8546005c02 100644 --- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td +++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td @@ -3785,8 +3785,7 @@ class Pseudo pattern> def Callseq_Start : NVPTXInst<(outs), (ins i32imm:$amt1, i32imm:$amt2), - "\\{ // callseq $amt1, $amt2\n" - "\t.reg .b32 temp_param_reg;", + "\\{ // callseq $amt1, $amt2", [(callseq_start timm:$amt1, timm:$amt2)]>; def Callseq_End : NVPTXInst<(outs), (ins i32imm:$amt1, i32imm:$amt2), diff --git a/llvm/test/CodeGen/NVPTX/dynamic_stackalloc.ll b/llvm/test/CodeGen/NVPTX/dynamic_stackalloc.ll index 09297fb819ce..ce81957f2a39 100644 --- a/llvm/test/CodeGen/NVPTX/dynamic_stackalloc.ll +++ b/llvm/test/CodeGen/NVPTX/dynamic_stackalloc.ll @@ -17,7 +17,6 @@ ; CHECK-32-NEXT: alloca.u32 %r[[ALLOCA:[0-9]]], %r[[SIZE3]], 16; ; CHECK-32-NEXT: cvta.local.u32 %r[[ALLOCA]], %r[[ALLOCA]]; ; CHECK-32-NEXT: { // callseq 0, 0 -; CHECK-32-NEXT: .reg .b32 temp_param_reg; ; CHECK-32-NEXT: .param .b32 param0; ; CHECK-32-NEXT: st.param.b32 [param0+0], %r[[ALLOCA]]; @@ -27,7 +26,6 @@ ; CHECK-64-NEXT: alloca.u64 %rd[[ALLOCA:[0-9]]], %rd[[SIZE3]], 16; ; CHECK-64-NEXT: cvta.local.u64 %rd[[ALLOCA]], %rd[[ALLOCA]]; ; CHECK-64-NEXT: { // callseq 0, 0 -; CHECK-64-NEXT: .reg .b32 temp_param_reg; ; CHECK-64-NEXT: .param .b64 param0; ; CHECK-64-NEXT: st.param.b64 [param0+0], %rd[[ALLOCA]]; diff --git a/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll b/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll index 6895699a1dfe..96a4359d0ec4 100644 --- a/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll +++ b/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll @@ -827,7 +827,6 @@ define <4 x i8> @test_call(<4 x i8> %a, <4 x i8> %b) #0 { ; CHECK-NEXT: ld.param.u32 %r2, [test_call_param_1]; ; CHECK-NEXT: ld.param.u32 %r1, [test_call_param_0]; ; CHECK-NEXT: { // callseq 0, 0 -; CHECK-NEXT: .reg .b32 temp_param_reg; ; CHECK-NEXT: .param .align 4 .b8 param0[4]; ; CHECK-NEXT: st.param.b32 [param0+0], %r1; ; CHECK-NEXT: .param .align 4 .b8 param1[4]; @@ -856,7 +855,6 @@ define <4 x i8> @test_call_flipped(<4 x i8> %a, <4 x i8> %b) #0 { ; CHECK-NEXT: ld.param.u32 %r2, [test_call_flipped_param_1]; ; CHECK-NEXT: ld.param.u32 %r1, [test_call_flipped_param_0]; ; CHECK-NEXT: { // callseq 1, 0 -; CHECK-NEXT: .reg .b32 temp_param_reg; ; CHECK-NEXT: .param .align 4 .b8 param0[4]; ; CHECK-NEXT: st.param.b32 [param0+0], %r2; ; CHECK-NEXT: .param .align 4 .b8 param1[4]; @@ -885,7 +883,6 @@ define <4 x i8> @test_tailcall_flipped(<4 x i8> %a, <4 x i8> %b) #0 { ; CHECK-NEXT: ld.param.u32 %r2, [test_tailcall_flipped_param_1]; ; CHECK-NEXT: ld.param.u32 %r1, [test_tailcall_flipped_param_0]; ; CHECK-NEXT: { // callseq 2, 0 -; CHECK-NEXT: .reg .b32 temp_param_reg; ; CHECK-NEXT: .param .align 4 .b8 param0[4]; ; CHECK-NEXT: st.param.b32 [param0+0], %r2; ; CHECK-NEXT: .param .align 4 .b8 param1[4]; diff --git a/llvm/test/DebugInfo/NVPTX/dbg-declare-alloca.ll b/llvm/test/DebugInfo/NVPTX/dbg-declare-alloca.ll index 8d7608ead38e..de367dfa4acb 100644 --- a/llvm/test/DebugInfo/NVPTX/dbg-declare-alloca.ll +++ b/llvm/test/DebugInfo/NVPTX/dbg-declare-alloca.ll @@ -9,7 +9,6 @@ ; CHECK: add.u64 %rd1, %SP, 0; ; CHECK: .loc 1 5 3 // t.c:5:3 ; CHECK: { // callseq 0, 0 -; CHECK: .reg .b32 temp_param_reg; ; CHECK: .param .b64 param0; ; CHECK: st.param.b64 [param0+0], %rd1; ; CHECK: call.uni diff --git a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/nvptx-basic.ll.expected b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/nvptx-basic.ll.expected index a65f140b4601..3ac63d070933 100644 --- a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/nvptx-basic.ll.expected +++ b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/nvptx-basic.ll.expected @@ -30,7 +30,6 @@ define dso_local void @caller_St8x4(ptr nocapture noundef readonly byval(%struct ; CHECK-NEXT: ld.u64 %rd7, [%SP+24]; ; CHECK-NEXT: ld.u64 %rd8, [%SP+16]; ; CHECK-NEXT: { // callseq 0, 0 -; CHECK-NEXT: .reg .b32 temp_param_reg; ; CHECK-NEXT: .param .align 16 .b8 param0[32]; ; CHECK-NEXT: st.param.v2.b64 [param0+0], {%rd6, %rd5}; ; CHECK-NEXT: st.param.v2.b64 [param0+16], {%rd8, %rd7}; -- GitLab From 800f1050e190430f217e1fd0db9414dacc835e11 Mon Sep 17 00:00:00 2001 From: Shilei Tian Date: Wed, 17 Apr 2024 14:53:01 -0400 Subject: [PATCH 092/862] [GitHub] Add a new mapping for `offload` subproject (#89118) Fix #89071. --- .github/new-prs-labeler.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/new-prs-labeler.yml b/.github/new-prs-labeler.yml index 1502d64a7d3e..9cf64417d3cb 100644 --- a/.github/new-prs-labeler.yml +++ b/.github/new-prs-labeler.yml @@ -944,3 +944,6 @@ openmp:libomptarget: bazel: - utils/bazel/** + +offload: + - offload/** -- GitLab From 6f7976c883da592f2cf6bfadef152e4203c00445 Mon Sep 17 00:00:00 2001 From: Mark de Wever Date: Wed, 17 Apr 2024 20:55:51 +0200 Subject: [PATCH 093/862] [libc++][TZDB] Adds sys_info formatter. (#85896) Implements parts of: - P0355 Extending to Calendars and Time Zones - P1361 Integration of chrono with text formatting --- libcxx/docs/Status/FormatPaper.csv | 2 +- libcxx/include/__chrono/convert_to_tm.h | 5 + libcxx/include/__chrono/formatter.h | 45 +++++- libcxx/include/__chrono/ostream.h | 20 +++ libcxx/include/__chrono/sys_info.h | 2 +- libcxx/include/chrono | 5 + .../time.zone.info.sys/ostream.pass.cpp | 74 ++++++++++ .../time/time.syn/formatter.sys_info.pass.cpp | 137 ++++++++++++++++++ .../time.zone.info.sys/ostream.pass.cpp | 52 +++++++ .../concept.formattable.compile.pass.cpp | 4 +- libcxx/test/support/test_macros.h | 8 + 11 files changed, 347 insertions(+), 7 deletions(-) create mode 100644 libcxx/test/libcxx/time/time.zone/time.zone.info/time.zone.info.sys/ostream.pass.cpp create mode 100644 libcxx/test/std/time/time.syn/formatter.sys_info.pass.cpp create mode 100644 libcxx/test/std/time/time.zone/time.zone.info/time.zone.info.sys/ostream.pass.cpp diff --git a/libcxx/docs/Status/FormatPaper.csv b/libcxx/docs/Status/FormatPaper.csv index e9d407e79e25..8ace18815f53 100644 --- a/libcxx/docs/Status/FormatPaper.csv +++ b/libcxx/docs/Status/FormatPaper.csv @@ -24,7 +24,7 @@ Section,Description,Dependencies,Assignee,Status,First released version `[time.syn] `_,"Formatter ``chrono::year_month_weekday``",,Mark de Wever,|Complete|,16.0 `[time.syn] `_,"Formatter ``chrono::year_month_weekday_last``",,Mark de Wever,|Complete|,16.0 `[time.syn] `_,"Formatter ``chrono::hh_mm_ss>``",,Mark de Wever,|Complete|,17.0 -`[time.syn] `_,"Formatter ``chrono::sys_info``",A ```` implementation,Mark de Wever,, +`[time.syn] `_,"Formatter ``chrono::sys_info``",,Mark de Wever,|Complete|,19.0 `[time.syn] `_,"Formatter ``chrono::local_info``",A ```` implementation,Mark de Wever,, `[time.syn] `_,"Formatter ``chrono::zoned_time``",A ```` implementation,Mark de Wever,, diff --git a/libcxx/include/__chrono/convert_to_tm.h b/libcxx/include/__chrono/convert_to_tm.h index 1301cd6f1f1a..d2c5cf922ba6 100644 --- a/libcxx/include/__chrono/convert_to_tm.h +++ b/libcxx/include/__chrono/convert_to_tm.h @@ -20,6 +20,7 @@ #include <__chrono/month_weekday.h> #include <__chrono/monthday.h> #include <__chrono/statically_widen.h> +#include <__chrono/sys_info.h> #include <__chrono/system_clock.h> #include <__chrono/time_point.h> #include <__chrono/weekday.h> @@ -171,6 +172,10 @@ _LIBCPP_HIDE_FROM_ABI _Tm __convert_to_tm(const _ChronoT& __value) { if (__value.hours().count() > std::numeric_limits::max()) std::__throw_format_error("Formatting hh_mm_ss, encountered an hour overflow"); __result.tm_hour = __value.hours().count(); +# if !defined(_LIBCPP_HAS_NO_INCOMPLETE_TZDB) + } else if constexpr (same_as<_ChronoT, chrono::sys_info>) { + // Has no time information. +# endif } else static_assert(sizeof(_ChronoT) == 0, "Add the missing type specialization"); diff --git a/libcxx/include/__chrono/formatter.h b/libcxx/include/__chrono/formatter.h index f76e7b2ea0e8..0196deb8c1ff 100644 --- a/libcxx/include/__chrono/formatter.h +++ b/libcxx/include/__chrono/formatter.h @@ -24,6 +24,7 @@ #include <__chrono/ostream.h> #include <__chrono/parser_std_format_spec.h> #include <__chrono/statically_widen.h> +#include <__chrono/sys_info.h> #include <__chrono/system_clock.h> #include <__chrono/time_point.h> #include <__chrono/weekday.h> @@ -186,10 +187,11 @@ __format_zone_offset(basic_stringstream<_CharT>& __sstr, chrono::seconds __offse chrono::hh_mm_ss __hms{__offset}; std::ostreambuf_iterator<_CharT> __out_it{__sstr}; + // Note HMS does not allow formatting hours > 23, but the offset is not limited to 24H. + std::format_to(__out_it, _LIBCPP_STATICALLY_WIDEN(_CharT, "{:02}"), __hms.hours().count()); if (__modifier) - std::format_to(__out_it, _LIBCPP_STATICALLY_WIDEN(_CharT, "{:%H:%M}"), __hms); - else - std::format_to(__out_it, _LIBCPP_STATICALLY_WIDEN(_CharT, "{:%H%M}"), __hms); + __sstr << _CharT(':'); + std::format_to(__out_it, _LIBCPP_STATICALLY_WIDEN(_CharT, "{:02}"), __hms.minutes().count()); } // Helper to store the time zone information needed for formatting. @@ -202,7 +204,12 @@ struct _LIBCPP_HIDE_FROM_ABI __time_zone { template _LIBCPP_HIDE_FROM_ABI __time_zone __convert_to_time_zone([[maybe_unused]] const _Tp& __value) { - return {"UTC", chrono::seconds{0}}; +# if !defined(_LIBCPP_HAS_NO_INCOMPLETE_TZDB) + if constexpr (same_as<_Tp, chrono::sys_info>) + return {__value.abbrev, __value.offset}; + else +# endif + return {"UTC", chrono::seconds{0}}; } template @@ -409,6 +416,10 @@ _LIBCPP_HIDE_FROM_ABI constexpr bool __weekday_ok(const _Tp& __value) { return __value.weekday().ok(); else if constexpr (__is_hh_mm_ss<_Tp>) return true; +# if !defined(_LIBCPP_HAS_NO_INCOMPLETE_TZDB) + else if constexpr (same_as<_Tp, chrono::sys_info>) + return true; +# endif else static_assert(sizeof(_Tp) == 0, "Add the missing type specialization"); } @@ -449,6 +460,10 @@ _LIBCPP_HIDE_FROM_ABI constexpr bool __weekday_name_ok(const _Tp& __value) { return __value.weekday().ok(); else if constexpr (__is_hh_mm_ss<_Tp>) return true; +# if !defined(_LIBCPP_HAS_NO_INCOMPLETE_TZDB) + else if constexpr (same_as<_Tp, chrono::sys_info>) + return true; +# endif else static_assert(sizeof(_Tp) == 0, "Add the missing type specialization"); } @@ -489,6 +504,10 @@ _LIBCPP_HIDE_FROM_ABI constexpr bool __date_ok(const _Tp& __value) { return __value.ok(); else if constexpr (__is_hh_mm_ss<_Tp>) return true; +# if !defined(_LIBCPP_HAS_NO_INCOMPLETE_TZDB) + else if constexpr (same_as<_Tp, chrono::sys_info>) + return true; +# endif else static_assert(sizeof(_Tp) == 0, "Add the missing type specialization"); } @@ -529,6 +548,10 @@ _LIBCPP_HIDE_FROM_ABI constexpr bool __month_name_ok(const _Tp& __value) { return __value.month().ok(); else if constexpr (__is_hh_mm_ss<_Tp>) return true; +# if !defined(_LIBCPP_HAS_NO_INCOMPLETE_TZDB) + else if constexpr (same_as<_Tp, chrono::sys_info>) + return true; +# endif else static_assert(sizeof(_Tp) == 0, "Add the missing type specialization"); } @@ -858,6 +881,20 @@ public: return _Base::__parse(__ctx, __format_spec::__fields_chrono, __format_spec::__flags::__time); } }; + +# if !defined(_LIBCPP_HAS_NO_INCOMPLETE_TZDB) +template <__fmt_char_type _CharT> +struct formatter : public __formatter_chrono<_CharT> { +public: + using _Base = __formatter_chrono<_CharT>; + + template + _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) { + return _Base::__parse(__ctx, __format_spec::__fields_chrono, __format_spec::__flags::__time_zone); + } +}; +# endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_TZDB) + #endif // if _LIBCPP_STD_VER >= 20 _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__chrono/ostream.h b/libcxx/include/__chrono/ostream.h index b687ef8059d5..9476406d7610 100644 --- a/libcxx/include/__chrono/ostream.h +++ b/libcxx/include/__chrono/ostream.h @@ -19,6 +19,7 @@ #include <__chrono/month_weekday.h> #include <__chrono/monthday.h> #include <__chrono/statically_widen.h> +#include <__chrono/sys_info.h> #include <__chrono/system_clock.h> #include <__chrono/weekday.h> #include <__chrono/year.h> @@ -262,6 +263,25 @@ operator<<(basic_ostream<_CharT, _Traits>& __os, const hh_mm_ss<_Duration> __hms return __os << std::format(__os.getloc(), _LIBCPP_STATICALLY_WIDEN(_CharT, "{:L%T}"), __hms); } +# if !defined(_LIBCPP_HAS_NO_INCOMPLETE_TZDB) + +template +_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>& +operator<<(basic_ostream<_CharT, _Traits>& __os, const sys_info& __info) { + // __info.abbrev is always std::basic_string. + // Since these strings typically are short the conversion should be cheap. + std::basic_string<_CharT> __abbrev{__info.abbrev.begin(), __info.abbrev.end()}; + return __os << std::format( + _LIBCPP_STATICALLY_WIDEN(_CharT, "[{:%F %T}, {:%F %T}) {:%T} {:%Q%q} \"{}\""), + __info.begin, + __info.end, + hh_mm_ss{__info.offset}, + __info.save, + __abbrev); +} + +# endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_TZDB) + } // namespace chrono #endif // if _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__chrono/sys_info.h b/libcxx/include/__chrono/sys_info.h index 794d22f2ccc1..461d5322d413 100644 --- a/libcxx/include/__chrono/sys_info.h +++ b/libcxx/include/__chrono/sys_info.h @@ -42,7 +42,7 @@ struct sys_info { } // namespace chrono -# endif //_LIBCPP_STD_VER >= 20 +# endif // _LIBCPP_STD_VER >= 20 _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/chrono b/libcxx/include/chrono index 513ae52006e8..f0275ec1eba7 100644 --- a/libcxx/include/chrono +++ b/libcxx/include/chrono @@ -733,6 +733,10 @@ struct sys_info { string abbrev; }; +template // C++20 + basic_ostream& + operator<<(basic_ostream& os, const sys_info& si); + // 25.10.5, class time_zone // C++20 enum class choose {earliest, latest}; class time_zone { @@ -829,6 +833,7 @@ namespace std { template struct formatter; // C++20 template struct formatter>, charT>; // C++20 + template struct formatter; // C++20 } // namespace std namespace chrono { diff --git a/libcxx/test/libcxx/time/time.zone/time.zone.info/time.zone.info.sys/ostream.pass.cpp b/libcxx/test/libcxx/time/time.zone/time.zone.info/time.zone.info.sys/ostream.pass.cpp new file mode 100644 index 000000000000..faa7d855c8de --- /dev/null +++ b/libcxx/test/libcxx/time/time.zone/time.zone.info/time.zone.info.sys/ostream.pass.cpp @@ -0,0 +1,74 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17 +// UNSUPPORTED: no-localization + +// TODO FMT This test should not require std::to_chars(floating-point) +// XFAIL: availability-fp_to_chars-missing + +// XFAIL: libcpp-has-no-incomplete-tzdb + +// + +// template +// basic_ostream& +// operator<<(basic_ostream& os, const sys_info& r); + +// [time.zone.info.sys] +// 7 Effects: Streams out the sys_info object r in an unspecified format. +// 8 Returns: os. +// +// Tests the output produced by this function. + +#include +#include +#include +#include + +#include "assert_macros.h" +#include "test_macros.h" +#include "make_string.h" +#include "concat_macros.h" + +#define SV(S) MAKE_STRING_VIEW(CharT, S) + +template +static void test(std::basic_string_view expected, std::chrono::sys_info&& info) { + std::basic_stringstream sstr; + sstr << info; + std::basic_string output = sstr.str(); + + TEST_REQUIRE(expected == output, + TEST_WRITE_CONCATENATED("\nExpected output ", expected, "\nActual output ", output, '\n')); +} + +template +static void test() { + using namespace std::literals::chrono_literals; + namespace tz = std::chrono; + + test(SV("[-10484-10-16 15:30:08, 14423-03-17 15:30:07) 00:00:00 0min \"TZ\""), + tz::sys_info{tz::sys_seconds::min(), tz::sys_seconds::max(), 0s, 0min, "TZ"}); + + test(SV("[1970-01-01 00:00:00, 2038-12-31 00:00:00) 12:23:45 -67min \"DMY\""), + tz::sys_info{static_cast(tz::year_month_day{1970y, tz::January, 1d}), + static_cast(tz::year_month_day{2038y, tz::December, 31d}), + 12h + 23min + 45s, + -67min, + "DMY"}); +} + +int main(int, const char**) { + test(); +#ifndef TEST_HAS_NO_WIDE_CHARACTERS + test(); +#endif + + return 0; +} diff --git a/libcxx/test/std/time/time.syn/formatter.sys_info.pass.cpp b/libcxx/test/std/time/time.syn/formatter.sys_info.pass.cpp new file mode 100644 index 000000000000..0a41424cfdcb --- /dev/null +++ b/libcxx/test/std/time/time.syn/formatter.sys_info.pass.cpp @@ -0,0 +1,137 @@ +//===----------------------------------------------------------------------===// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17 +// UNSUPPORTED: no-localization + +// TODO FMT This test should not require std::to_chars(floating-point) +// XFAIL: availability-fp_to_chars-missing + +// XFAIL: libcpp-has-no-incomplete-tzdb + +// REQUIRES: locale.fr_FR.UTF-8 +// REQUIRES: locale.ja_JP.UTF-8 + +// +// +// template struct formatter; + +#include +#include + +#include +#include +#include +#include +#include + +#include "formatter_tests.h" +#include "make_string.h" +#include "platform_support.h" // locale name macros +#include "test_macros.h" + +template +static void test_no_chrono_specs() { +// This test libc++ specific due to +// [time.zone.info.sys]/7 +// Effects: Streams out the sys_info object r in an unspecified format. +#ifdef _LIBCPP_VERSION + using namespace std::literals::chrono_literals; + namespace tz = std::chrono; + + std::locale::global(std::locale(LOCALE_fr_FR_UTF_8)); + + // Non localized output + + check(SV("[-10484-10-16 15:30:08, 14423-03-17 15:30:07) 00:00:00 0min \"TZ\""), + SV("{}"), + tz::sys_info{tz::sys_seconds::min(), tz::sys_seconds::max(), 0s, 0min, "TZ"}); + + check(SV("[1970-01-01 00:00:00, 2038-12-31 00:00:00) 12:23:45 -67min \"DMY\""), + SV("{}"), + tz::sys_info{static_cast(tz::year_month_day{1970y, tz::January, 1d}), + static_cast(tz::year_month_day{2038y, tz::December, 31d}), + 12h + 23min + 45s, + -67min, + "DMY"}); + + std::locale::global(std::locale::classic()); +#endif // _LIBCPP_VERSION +} + +template +static void test_valid_values() { + using namespace std::literals::chrono_literals; + + constexpr std::basic_string_view fmt = SV("{:%%z='%z'%t%%Ez='%Ez'%t%%Oz='%Oz'%t%%Z='%Z'%n}"); + constexpr std::basic_string_view lfmt = SV("{:L%%z='%z'%t%%Ez='%Ez'%t%%Oz='%Oz'%t%%Z='%Z'%n}"); + + const std::locale loc(LOCALE_ja_JP_UTF_8); + std::locale::global(std::locale(LOCALE_fr_FR_UTF_8)); + + // Non localized output using C-locale + check(SV("%z='-0200'\t%Ez='-02:00'\t%Oz='-02:00'\t%Z='NEG'\n"), + fmt, + std::chrono::sys_info{std::chrono::sys_seconds{0s}, std::chrono::sys_seconds{0s}, -2h, 0min, "NEG"}); + + check(SV("%z='+0000'\t%Ez='+00:00'\t%Oz='+00:00'\t%Z='ZERO'\n"), + fmt, + std::chrono::sys_info{std::chrono::sys_seconds{0s}, std::chrono::sys_seconds{0s}, 0s, 0min, "ZERO"}); + + check(SV("%z='+1115'\t%Ez='+11:15'\t%Oz='+11:15'\t%Z='POS'\n"), + fmt, + std::chrono::sys_info{std::chrono::sys_seconds{0s}, std::chrono::sys_seconds{0s}, 11h + 15min, 0min, "POS"}); + + // Use the global locale (fr_FR) + check(SV("%z='-0200'\t%Ez='-02:00'\t%Oz='-02:00'\t%Z='NEG'\n"), + lfmt, + std::chrono::sys_info{std::chrono::sys_seconds{0s}, std::chrono::sys_seconds{0s}, -2h, 0min, "NEG"}); + + check(SV("%z='+0000'\t%Ez='+00:00'\t%Oz='+00:00'\t%Z='ZERO'\n"), + lfmt, + std::chrono::sys_info{std::chrono::sys_seconds{0s}, std::chrono::sys_seconds{0s}, 0s, 0min, "ZERO"}); + + check(SV("%z='+1115'\t%Ez='+11:15'\t%Oz='+11:15'\t%Z='POS'\n"), + lfmt, + std::chrono::sys_info{std::chrono::sys_seconds{0s}, std::chrono::sys_seconds{0s}, 11h + 15min, 0min, "POS"}); + + // Use supplied locale (ja_JP). + check(loc, + SV("%z='-0200'\t%Ez='-02:00'\t%Oz='-02:00'\t%Z='NEG'\n"), + lfmt, + std::chrono::sys_info{std::chrono::sys_seconds{0s}, std::chrono::sys_seconds{0s}, -2h, 0min, "NEG"}); + + check(loc, + SV("%z='+0000'\t%Ez='+00:00'\t%Oz='+00:00'\t%Z='ZERO'\n"), + lfmt, + std::chrono::sys_info{std::chrono::sys_seconds{0s}, std::chrono::sys_seconds{0s}, 0s, 0min, "ZERO"}); + + check(loc, + SV("%z='+1115'\t%Ez='+11:15'\t%Oz='+11:15'\t%Z='POS'\n"), + lfmt, + std::chrono::sys_info{std::chrono::sys_seconds{0s}, std::chrono::sys_seconds{0s}, 11h + 15min, 0min, "POS"}); + + std::locale::global(std::locale::classic()); +} + +template +static void test() { + test_no_chrono_specs(); + test_valid_values(); + + check_invalid_types({SV("z"), SV("Z"), SV("Ez"), SV("Oz")}, std::chrono::sys_info{}); +} + +int main(int, char**) { + test(); + +#ifndef TEST_HAS_NO_WIDE_CHARACTERS + test(); +#endif + + return 0; +} diff --git a/libcxx/test/std/time/time.zone/time.zone.info/time.zone.info.sys/ostream.pass.cpp b/libcxx/test/std/time/time.zone/time.zone.info/time.zone.info.sys/ostream.pass.cpp new file mode 100644 index 000000000000..82c4844b423c --- /dev/null +++ b/libcxx/test/std/time/time.zone/time.zone.info/time.zone.info.sys/ostream.pass.cpp @@ -0,0 +1,52 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17 +// UNSUPPORTED: no-localization + +// TODO FMT This test should not require std::to_chars(floating-point) +// XFAIL: availability-fp_to_chars-missing + +// XFAIL: libcpp-has-no-incomplete-tzdb + +// + +// template +// basic_ostream& +// operator<<(basic_ostream& os, const sys_info& r); + +// [time.zone.info.sys] +// 7 Effects: Streams out the sys_info object r in an unspecified format. +// 8 Returns: os. +// +// There is a private libc++ test that validates the exact output. + +#include +#include +#include +#include + +#include "test_macros.h" + +template +static void test() { + using namespace std::literals::chrono_literals; + std::chrono::sys_info s{std::chrono::sys_seconds{0s}, std::chrono::sys_seconds{0s}, 0h, 0min, ""}; + std::basic_ostringstream os; + std::basic_ostream& result = std::chrono::operator<<(os, s); + assert(std::addressof(result) == std::addressof(os)); +} + +int main(int, const char**) { + test(); +#ifndef TEST_HAS_NO_WIDE_CHARACTERS + test(); +#endif + + return 0; +} diff --git a/libcxx/test/std/utilities/format/format.formattable/concept.formattable.compile.pass.cpp b/libcxx/test/std/utilities/format/format.formattable/concept.formattable.compile.pass.cpp index fad5277fd6e1..f40784ec446f 100644 --- a/libcxx/test/std/utilities/format/format.formattable/concept.formattable.compile.pass.cpp +++ b/libcxx/test/std/utilities/format/format.formattable/concept.formattable.compile.pass.cpp @@ -176,10 +176,12 @@ void test_P1361() { assert_is_formattable, CharT>(); - //assert_is_formattable(); +# if !defined(TEST_HAS_NO_INCOMPLETE_TZDB) + assert_is_formattable(); //assert_is_formattable(); //assert_is_formattable(); +# endif // !defined(TEST_HAS_NO_INCOMPLETE_TZDB) #endif // TEST_HAS_NO_LOCALIZATION } diff --git a/libcxx/test/support/test_macros.h b/libcxx/test/support/test_macros.h index 7b2dcbb52d0c..fe9207d7e596 100644 --- a/libcxx/test/support/test_macros.h +++ b/libcxx/test/support/test_macros.h @@ -417,6 +417,14 @@ inline Tp const& DoNotOptimize(Tp const& value) { # define TEST_HAS_NO_RANDOM_DEVICE #endif +#if defined(_LIBCPP_HAS_NO_INCOMPLETE_TZDB) +# define TEST_HAS_NO_INCOMPLETE_TZDB +#endif + +#if defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) +# define TEST_HAS_NO_TIME_ZONE_DATABASE +#endif + #if defined(TEST_COMPILER_CLANG) # define TEST_DIAGNOSTIC_PUSH _Pragma("clang diagnostic push") # define TEST_DIAGNOSTIC_POP _Pragma("clang diagnostic pop") -- GitLab From b1dc62f139ef265a36a2a739ce9ba4e1e48a6dbe Mon Sep 17 00:00:00 2001 From: Amirreza Ashouri Date: Wed, 17 Apr 2024 22:28:11 +0330 Subject: [PATCH 094/862] [clang]Treat arguments to builtin type traits as template type arguments (#87132) This change improves error messages for builtins in case of empty parentheses. Fixes llvm#86997 --- clang/lib/Parse/ParseExprCXX.cpp | 12 ++++++------ clang/test/Sema/static-assert.c | 9 ++++----- clang/test/SemaCXX/builtins.cpp | 5 +++++ clang/test/SemaCXX/deprecated-builtins.cpp | 5 +++++ 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/clang/lib/Parse/ParseExprCXX.cpp b/clang/lib/Parse/ParseExprCXX.cpp index 43d6105dcf31..0d2ad980696f 100644 --- a/clang/lib/Parse/ParseExprCXX.cpp +++ b/clang/lib/Parse/ParseExprCXX.cpp @@ -3910,10 +3910,10 @@ ExprResult Parser::ParseTypeTrait() { SmallVector Args; do { // Parse the next type. - TypeResult Ty = - ParseTypeName(/*SourceRange=*/nullptr, - getLangOpts().CPlusPlus ? DeclaratorContext::TemplateArg - : DeclaratorContext::TypeName); + TypeResult Ty = ParseTypeName(/*SourceRange=*/nullptr, + getLangOpts().CPlusPlus + ? DeclaratorContext::TemplateTypeArg + : DeclaratorContext::TypeName); if (Ty.isInvalid()) { Parens.skipToEnd(); return ExprError(); @@ -3955,8 +3955,8 @@ ExprResult Parser::ParseArrayTypeTrait() { if (T.expectAndConsume()) return ExprError(); - TypeResult Ty = - ParseTypeName(/*SourceRange=*/nullptr, DeclaratorContext::TemplateArg); + TypeResult Ty = ParseTypeName(/*SourceRange=*/nullptr, + DeclaratorContext::TemplateTypeArg); if (Ty.isInvalid()) { SkipUntil(tok::comma, StopAtSemi); SkipUntil(tok::r_paren, StopAtSemi); diff --git a/clang/test/Sema/static-assert.c b/clang/test/Sema/static-assert.c index ae5e8076e0be..4e9e6b7ee558 100644 --- a/clang/test/Sema/static-assert.c +++ b/clang/test/Sema/static-assert.c @@ -1,6 +1,6 @@ -// RUN: %clang_cc1 -std=c11 -Wgnu-folding-constant -fsyntax-only -verify=expected,c %s -// RUN: %clang_cc1 -fms-compatibility -Wgnu-folding-constant -DMS -fsyntax-only -verify=expected,ms,c %s -// RUN: %clang_cc1 -std=c99 -pedantic -Wgnu-folding-constant -fsyntax-only -verify=expected,ext,c %s +// RUN: %clang_cc1 -std=c11 -Wgnu-folding-constant -fsyntax-only -verify %s +// RUN: %clang_cc1 -fms-compatibility -Wgnu-folding-constant -DMS -fsyntax-only -verify=expected,ms %s +// RUN: %clang_cc1 -std=c99 -pedantic -Wgnu-folding-constant -fsyntax-only -verify=expected,ext %s // RUN: %clang_cc1 -xc++ -std=c++11 -pedantic -fsyntax-only -verify=expected,ext,cxx %s _Static_assert("foo", "string is nonzero"); // ext-warning {{'_Static_assert' is a C11 extension}} @@ -57,8 +57,7 @@ UNION(char[2], short) u2 = { .one = { 'a', 'b' } }; // ext-warning 3 {{'_Static_ typedef UNION(char, short) U3; // expected-error {{static assertion failed due to requirement 'sizeof(char) == sizeof(short)': type size mismatch}} \ // expected-note{{evaluates to '1 == 2'}} \ // ext-warning 3 {{'_Static_assert' is a C11 extension}} -typedef UNION(float, 0.5f) U4; // c-error {{expected a type}} \ - // cxx-error {{type name requires a specifier or qualifier}} \ +typedef UNION(float, 0.5f) U4; // expected-error {{expected a type}} \ // ext-warning 3 {{'_Static_assert' is a C11 extension}} // After defining the assert macro in MS-compatibility mode, we should diff --git a/clang/test/SemaCXX/builtins.cpp b/clang/test/SemaCXX/builtins.cpp index 567094c94c17..080b4476c7ee 100644 --- a/clang/test/SemaCXX/builtins.cpp +++ b/clang/test/SemaCXX/builtins.cpp @@ -76,6 +76,11 @@ using ConstMemFnType = int (Dummy::*)() const; void foo() {} +void test_builtin_empty_parentheses_diags() { + __is_trivially_copyable(); // expected-error {{expected a type}} + __is_trivially_copyable(1); // expected-error {{expected a type}} +} + void test_builtin_launder_diags(void *vp, const void *cvp, FnType *fnp, MemFnType mfp, ConstMemFnType cmfp, int (&Arr)[5]) { __builtin_launder(vp); // expected-error {{void pointer argument to '__builtin_launder' is not allowed}} diff --git a/clang/test/SemaCXX/deprecated-builtins.cpp b/clang/test/SemaCXX/deprecated-builtins.cpp index 849b9b014fff..fafc1da4da13 100644 --- a/clang/test/SemaCXX/deprecated-builtins.cpp +++ b/clang/test/SemaCXX/deprecated-builtins.cpp @@ -17,3 +17,8 @@ void f() { a = __has_trivial_destructor(A); // expected-warning-re {{__has_trivial_destructor {{.*}} use __is_trivially_destructible}} } + +void test_builtin_empty_parentheses_diags(void) { + __has_nothrow_copy(); // expected-error {{expected a type}} + __has_nothrow_copy(1); // expected-error {{expected a type}} +} -- GitLab From 6cea7c491f4c4c68aa0494a9b18f36ff40c22c81 Mon Sep 17 00:00:00 2001 From: Arthur Eubanks Date: Wed, 17 Apr 2024 12:04:18 -0700 Subject: [PATCH 095/862] [X86] Always use 64-bit relocations in no-PIC large code model (#89101) This matches other types of relocations, e.g. to constant pool. And makes things more consistent with PIC large code model. Some users of the large code model may not place small data in the lower 2GB of the address space (e.g. https://github.com/ClangBuiltLinux/linux/issues/2016), so just unconditionally use 64-bit relocations in the large code model. So now functions in a section not marked large will use 64-bit relocations to reference everything when using the large code model. This also fixes some lldb tests broken by #88172 (https://lab.llvm.org/buildbot/#/builders/68/builds/72458). --- llvm/lib/Target/X86/X86ISelDAGToDAG.cpp | 7 +++---- llvm/test/CodeGen/X86/code-model-elf.ll | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp index 4e4241efd63d..7dcde2a50894 100644 --- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp +++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp @@ -2927,11 +2927,10 @@ bool X86DAGToDAGISel::selectAddr(SDNode *Parent, SDValue N, SDValue &Base, } bool X86DAGToDAGISel::selectMOV64Imm32(SDValue N, SDValue &Imm) { - // Cannot use 32 bit constants to reference objects in kernel code model. - // Cannot use 32 bit constants to reference objects in large PIC mode since - // GOTOFF is 64 bits. + // Cannot use 32 bit constants to reference objects in kernel/large code + // model. if (TM.getCodeModel() == CodeModel::Kernel || - (TM.getCodeModel() == CodeModel::Large && TM.isPositionIndependent())) + TM.getCodeModel() == CodeModel::Large) return false; // In static codegen with small code model, we can get the address of a label diff --git a/llvm/test/CodeGen/X86/code-model-elf.ll b/llvm/test/CodeGen/X86/code-model-elf.ll index 0da62e3e7a65..f60f75bc2691 100644 --- a/llvm/test/CodeGen/X86/code-model-elf.ll +++ b/llvm/test/CodeGen/X86/code-model-elf.ll @@ -350,7 +350,7 @@ define dso_local ptr @lea_forced_small_data() #0 { ; ; LARGE-STATIC-LABEL: lea_forced_small_data: ; LARGE-STATIC: # %bb.0: -; LARGE-STATIC-NEXT: movl $forced_small_data, %eax +; LARGE-STATIC-NEXT: movabsq $forced_small_data, %rax ; LARGE-STATIC-NEXT: retq ; ; SMALL-PIC-LABEL: lea_forced_small_data: @@ -403,7 +403,7 @@ define dso_local i32 @load_forced_small_data() #0 { ; ; LARGE-STATIC-LABEL: load_forced_small_data: ; LARGE-STATIC: # %bb.0: -; LARGE-STATIC-NEXT: movl $forced_small_data+8, %eax +; LARGE-STATIC-NEXT: movabsq $forced_small_data+8, %rax ; LARGE-STATIC-NEXT: movl (%rax), %eax ; LARGE-STATIC-NEXT: retq ; -- GitLab From 1460b4964c7ada2f7536006722c8585b5bd0a1b5 Mon Sep 17 00:00:00 2001 From: Arthur Eubanks Date: Wed, 17 Apr 2024 19:11:58 +0000 Subject: [PATCH 096/862] [gn build] Manually port d423d80e560d --- llvm/utils/gn/secondary/libcxx/include/BUILD.gn | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn index ee44558a4e99..bd08ce044c24 100644 --- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn +++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn @@ -37,9 +37,9 @@ if (current_toolchain == default_toolchain) { "_LIBCPP_INSTRUMENTED_WITH_ASAN=", "_LIBCPP_ABI_DEFINES=", "_LIBCPP_HARDENING_MODE_DEFAULT=_LIBCPP_HARDENING_MODE_NONE", - "_LIBCPP_PSTL_CPU_BACKEND_LIBDISPATCH=", - "_LIBCPP_PSTL_CPU_BACKEND_SERIAL=1", - "_LIBCPP_PSTL_CPU_BACKEND_THREAD=", + "_LIBCPP_PSTL_BACKEND_LIBDISPATCH=", + "_LIBCPP_PSTL_BACKEND_SERIAL=1", + "_LIBCPP_PSTL_BACKEND_STD_THREAD=", ] if (libcxx_abi_version != 1) { values += [ "_LIBCPP_ABI_VERSION=$libcxx_abi_version" ] @@ -143,18 +143,12 @@ if (current_toolchain == default_toolchain) { "__algorithm/pop_heap.h", "__algorithm/prev_permutation.h", "__algorithm/pstl_any_all_none_of.h", - "__algorithm/pstl_backend.h", - "__algorithm/pstl_backends/cpu_backend.h", "__algorithm/pstl_backends/cpu_backends/any_of.h", - "__algorithm/pstl_backends/cpu_backends/backend.h", "__algorithm/pstl_backends/cpu_backends/fill.h", "__algorithm/pstl_backends/cpu_backends/find_if.h", "__algorithm/pstl_backends/cpu_backends/for_each.h", - "__algorithm/pstl_backends/cpu_backends/libdispatch.h", "__algorithm/pstl_backends/cpu_backends/merge.h", - "__algorithm/pstl_backends/cpu_backends/serial.h", "__algorithm/pstl_backends/cpu_backends/stable_sort.h", - "__algorithm/pstl_backends/cpu_backends/thread.h", "__algorithm/pstl_backends/cpu_backends/transform.h", "__algorithm/pstl_backends/cpu_backends/transform_reduce.h", "__algorithm/pstl_copy.h", @@ -664,6 +658,11 @@ if (current_toolchain == default_toolchain) { "__numeric/transform_exclusive_scan.h", "__numeric/transform_inclusive_scan.h", "__numeric/transform_reduce.h", + "__pstl/backends/libdispatch.h", + "__pstl/backends/serial.h", + "__pstl/backends/std_thread.h", + "__pstl/configuration.h", + "__pstl/configuration_fwd.h", "__pstl/cpu_algos/cpu_traits.h", "__random/bernoulli_distribution.h", "__random/binomial_distribution.h", -- GitLab From db2f64ee1f743153f242ca92bf627458ba7f8000 Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Wed, 17 Apr 2024 21:20:39 +0200 Subject: [PATCH 097/862] AMDGPU: Fix not handling atomicrmw fadd in exotic address spaces correctly We try to interpret unknown address space numbers as aliases of global, but this wasn't applied here. Also improve test coverage for the buffer fat pointer address space. --- llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 2 +- .../AMDGPU/expand-atomic-rmw-fadd.ll | 414 ++++++++++++++++++ 2 files changed, 415 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index 6fde73119f81..0c706d51cb66 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -16062,7 +16062,7 @@ SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const { if (HasSystemScope) return AtomicExpansionKind::CmpXChg; - if ((AS == AMDGPUAS::GLOBAL_ADDRESS || + if ((AMDGPU::isExtendedGlobalAddrSpace(AS) || AS == AMDGPUAS::BUFFER_FAT_POINTER) && Ty->isFloatTy()) { // global/buffer atomic fadd f32 no-rtn: gfx908, gfx90a, gfx940, gfx11+. diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll index a5d4c329446f..9010dec31c16 100644 --- a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll +++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll @@ -59,6 +59,112 @@ define void @test_atomicrmw_fadd_f32_global_no_use_unsafe(ptr addrspace(1) %ptr, ret void } +define void @test_atomicrmw_fadd_f32_buffer_fat_ptr_no_use_unsafe(ptr addrspace(7) %ptr, float %value) #0 { +; CI-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_no_use_unsafe( +; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(7) [[PTR:%.*]], align 4 +; CI-NEXT: br label [[ATOMICRMW_START:%.*]] +; CI: atomicrmw.start: +; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; CI-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; CI-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(7) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4 +; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; CI: atomicrmw.end: +; CI-NEXT: ret void +; +; GFX9-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_no_use_unsafe( +; GFX9-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(7) [[PTR:%.*]], align 4 +; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]] +; GFX9: atomicrmw.start: +; GFX9-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; GFX9-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; GFX9-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; GFX9-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(7) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4 +; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; GFX9: atomicrmw.end: +; GFX9-NEXT: ret void +; +; GFX908-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_no_use_unsafe( +; GFX908-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(7) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4 +; GFX908-NEXT: ret void +; +; GFX90A-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_no_use_unsafe( +; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(7) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4 +; GFX90A-NEXT: ret void +; +; GFX940-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_no_use_unsafe( +; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(7) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4 +; GFX940-NEXT: ret void +; +; GFX11-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_no_use_unsafe( +; GFX11-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(7) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4 +; GFX11-NEXT: ret void +; + %res = atomicrmw fadd ptr addrspace(7) %ptr, float %value syncscope("wavefront") monotonic + ret void +} + +define void @test_atomicrmw_fadd_f32_as999_no_use_unsafe(ptr addrspace(999) %ptr, float %value) #0 { +; CI-LABEL: @test_atomicrmw_fadd_f32_as999_no_use_unsafe( +; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(999) [[PTR:%.*]], align 4 +; CI-NEXT: br label [[ATOMICRMW_START:%.*]] +; CI: atomicrmw.start: +; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; CI-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; CI-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(999) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4 +; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; CI: atomicrmw.end: +; CI-NEXT: ret void +; +; GFX9-LABEL: @test_atomicrmw_fadd_f32_as999_no_use_unsafe( +; GFX9-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(999) [[PTR:%.*]], align 4 +; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]] +; GFX9: atomicrmw.start: +; GFX9-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; GFX9-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; GFX9-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; GFX9-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(999) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4 +; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; GFX9: atomicrmw.end: +; GFX9-NEXT: ret void +; +; GFX908-LABEL: @test_atomicrmw_fadd_f32_as999_no_use_unsafe( +; GFX908-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(999) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4 +; GFX908-NEXT: ret void +; +; GFX90A-LABEL: @test_atomicrmw_fadd_f32_as999_no_use_unsafe( +; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(999) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4 +; GFX90A-NEXT: ret void +; +; GFX940-LABEL: @test_atomicrmw_fadd_f32_as999_no_use_unsafe( +; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(999) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4 +; GFX940-NEXT: ret void +; +; GFX11-LABEL: @test_atomicrmw_fadd_f32_as999_no_use_unsafe( +; GFX11-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(999) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4 +; GFX11-NEXT: ret void +; + %res = atomicrmw fadd ptr addrspace(999) %ptr, float %value syncscope("wavefront") monotonic + ret void +} + define float @test_atomicrmw_fadd_f32_global_unsafe(ptr addrspace(1) %ptr, float %value) #0 { ; CI-LABEL: @test_atomicrmw_fadd_f32_global_unsafe( ; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 4 @@ -124,6 +230,136 @@ define float @test_atomicrmw_fadd_f32_global_unsafe(ptr addrspace(1) %ptr, float ret float %res } +define float @test_atomicrmw_fadd_f32_buffer_fat_ptr_unsafe(ptr addrspace(7) %ptr, float %value) #0 { +; CI-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_unsafe( +; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(7) [[PTR:%.*]], align 4 +; CI-NEXT: br label [[ATOMICRMW_START:%.*]] +; CI: atomicrmw.start: +; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; CI-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; CI-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(7) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4 +; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; CI: atomicrmw.end: +; CI-NEXT: ret float [[TMP5]] +; +; GFX9-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_unsafe( +; GFX9-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(7) [[PTR:%.*]], align 4 +; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]] +; GFX9: atomicrmw.start: +; GFX9-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; GFX9-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; GFX9-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; GFX9-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(7) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4 +; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; GFX9: atomicrmw.end: +; GFX9-NEXT: ret float [[TMP5]] +; +; GFX908-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_unsafe( +; GFX908-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(7) [[PTR:%.*]], align 4 +; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]] +; GFX908: atomicrmw.start: +; GFX908-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; GFX908-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; GFX908-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; GFX908-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(7) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4 +; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; GFX908: atomicrmw.end: +; GFX908-NEXT: ret float [[TMP5]] +; +; GFX90A-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_unsafe( +; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(7) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4 +; GFX90A-NEXT: ret float [[RES]] +; +; GFX940-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_unsafe( +; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(7) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4 +; GFX940-NEXT: ret float [[RES]] +; +; GFX11-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_unsafe( +; GFX11-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(7) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4 +; GFX11-NEXT: ret float [[RES]] +; + %res = atomicrmw fadd ptr addrspace(7) %ptr, float %value syncscope("wavefront") monotonic + ret float %res +} + +define float @test_atomicrmw_fadd_f32_as999_unsafe(ptr addrspace(999) %ptr, float %value) #0 { +; CI-LABEL: @test_atomicrmw_fadd_f32_as999_unsafe( +; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(999) [[PTR:%.*]], align 4 +; CI-NEXT: br label [[ATOMICRMW_START:%.*]] +; CI: atomicrmw.start: +; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; CI-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; CI-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(999) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4 +; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; CI: atomicrmw.end: +; CI-NEXT: ret float [[TMP5]] +; +; GFX9-LABEL: @test_atomicrmw_fadd_f32_as999_unsafe( +; GFX9-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(999) [[PTR:%.*]], align 4 +; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]] +; GFX9: atomicrmw.start: +; GFX9-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; GFX9-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; GFX9-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; GFX9-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(999) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4 +; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; GFX9: atomicrmw.end: +; GFX9-NEXT: ret float [[TMP5]] +; +; GFX908-LABEL: @test_atomicrmw_fadd_f32_as999_unsafe( +; GFX908-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(999) [[PTR:%.*]], align 4 +; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]] +; GFX908: atomicrmw.start: +; GFX908-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; GFX908-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; GFX908-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; GFX908-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(999) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4 +; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; GFX908: atomicrmw.end: +; GFX908-NEXT: ret float [[TMP5]] +; +; GFX90A-LABEL: @test_atomicrmw_fadd_f32_as999_unsafe( +; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(999) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4 +; GFX90A-NEXT: ret float [[RES]] +; +; GFX940-LABEL: @test_atomicrmw_fadd_f32_as999_unsafe( +; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(999) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4 +; GFX940-NEXT: ret float [[RES]] +; +; GFX11-LABEL: @test_atomicrmw_fadd_f32_as999_unsafe( +; GFX11-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(999) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4 +; GFX11-NEXT: ret float [[RES]] +; + %res = atomicrmw fadd ptr addrspace(999) %ptr, float %value syncscope("wavefront") monotonic + ret float %res +} + define double @test_atomicrmw_fadd_f64_global_unsafe(ptr addrspace(1) %ptr, double %value) #0 { ; CI-LABEL: @test_atomicrmw_fadd_f64_global_unsafe( ; CI-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR:%.*]], align 8 @@ -545,6 +781,184 @@ define float @test_atomicrmw_fadd_f32_global(ptr addrspace(1) %ptr, float %value ret float %res } +define float @test_atomicrmw_fadd_f32_buffer_fat_ptr(ptr addrspace(7) %ptr, float %value) { +; CI-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr( +; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(7) [[PTR:%.*]], align 4 +; CI-NEXT: br label [[ATOMICRMW_START:%.*]] +; CI: atomicrmw.start: +; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; CI-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; CI-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(7) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 4 +; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; CI: atomicrmw.end: +; CI-NEXT: ret float [[TMP5]] +; +; GFX9-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr( +; GFX9-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(7) [[PTR:%.*]], align 4 +; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]] +; GFX9: atomicrmw.start: +; GFX9-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; GFX9-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; GFX9-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; GFX9-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(7) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 4 +; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; GFX9: atomicrmw.end: +; GFX9-NEXT: ret float [[TMP5]] +; +; GFX908-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr( +; GFX908-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(7) [[PTR:%.*]], align 4 +; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]] +; GFX908: atomicrmw.start: +; GFX908-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; GFX908-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; GFX908-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; GFX908-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(7) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 4 +; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; GFX908: atomicrmw.end: +; GFX908-NEXT: ret float [[TMP5]] +; +; GFX90A-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr( +; GFX90A-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(7) [[PTR:%.*]], align 4 +; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]] +; GFX90A: atomicrmw.start: +; GFX90A-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; GFX90A-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; GFX90A-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; GFX90A-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(7) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 4 +; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; GFX90A-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; GFX90A: atomicrmw.end: +; GFX90A-NEXT: ret float [[TMP5]] +; +; GFX940-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr( +; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(7) [[PTR:%.*]], float [[VALUE:%.*]] seq_cst, align 4 +; GFX940-NEXT: ret float [[RES]] +; +; GFX11-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr( +; GFX11-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(7) [[PTR:%.*]], align 4 +; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]] +; GFX11: atomicrmw.start: +; GFX11-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; GFX11-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; GFX11-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; GFX11-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(7) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 4 +; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; GFX11-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; GFX11: atomicrmw.end: +; GFX11-NEXT: ret float [[TMP5]] +; + %res = atomicrmw fadd ptr addrspace(7) %ptr, float %value seq_cst + ret float %res +} + +define float @test_atomicrmw_fadd_f32_as999(ptr addrspace(999) %ptr, float %value) { +; CI-LABEL: @test_atomicrmw_fadd_f32_as999( +; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(999) [[PTR:%.*]], align 4 +; CI-NEXT: br label [[ATOMICRMW_START:%.*]] +; CI: atomicrmw.start: +; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; CI-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; CI-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(999) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 4 +; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; CI: atomicrmw.end: +; CI-NEXT: ret float [[TMP5]] +; +; GFX9-LABEL: @test_atomicrmw_fadd_f32_as999( +; GFX9-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(999) [[PTR:%.*]], align 4 +; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]] +; GFX9: atomicrmw.start: +; GFX9-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; GFX9-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; GFX9-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; GFX9-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(999) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 4 +; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; GFX9: atomicrmw.end: +; GFX9-NEXT: ret float [[TMP5]] +; +; GFX908-LABEL: @test_atomicrmw_fadd_f32_as999( +; GFX908-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(999) [[PTR:%.*]], align 4 +; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]] +; GFX908: atomicrmw.start: +; GFX908-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; GFX908-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; GFX908-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; GFX908-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(999) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 4 +; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; GFX908: atomicrmw.end: +; GFX908-NEXT: ret float [[TMP5]] +; +; GFX90A-LABEL: @test_atomicrmw_fadd_f32_as999( +; GFX90A-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(999) [[PTR:%.*]], align 4 +; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]] +; GFX90A: atomicrmw.start: +; GFX90A-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; GFX90A-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; GFX90A-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; GFX90A-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(999) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 4 +; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; GFX90A-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; GFX90A: atomicrmw.end: +; GFX90A-NEXT: ret float [[TMP5]] +; +; GFX940-LABEL: @test_atomicrmw_fadd_f32_as999( +; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(999) [[PTR:%.*]], float [[VALUE:%.*]] seq_cst, align 4 +; GFX940-NEXT: ret float [[RES]] +; +; GFX11-LABEL: @test_atomicrmw_fadd_f32_as999( +; GFX11-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(999) [[PTR:%.*]], align 4 +; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]] +; GFX11: atomicrmw.start: +; GFX11-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; GFX11-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; GFX11-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; GFX11-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(999) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 4 +; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; GFX11-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; GFX11: atomicrmw.end: +; GFX11-NEXT: ret float [[TMP5]] +; + %res = atomicrmw fadd ptr addrspace(999) %ptr, float %value seq_cst + ret float %res +} + define void @test_atomicrmw_fadd_f32_global_no_use_ieee(ptr addrspace(1) %ptr, float %value) { ; CI-LABEL: @test_atomicrmw_fadd_f32_global_no_use_ieee( ; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 4 -- GitLab From 652bcf685c72447f3cc46d93d6c9c1948e8499f3 Mon Sep 17 00:00:00 2001 From: Matthias Braun Date: Wed, 17 Apr 2024 12:48:02 -0700 Subject: [PATCH 098/862] CodeGenPrepare: Add support for llvm.threadlocal.address address-mode sinking (#87844) Depending on the TLSMode many thread-local accesses on x86 can be expressed by adding a %fs: segment register to an addressing mode. Even if there are mutliple users of a `llvm.threadlocal.address` intrinsic it is generally not worth sharing the value in a register but instead fold the %fs access into multiple addressing modes. Hence this changes CodeGenPrepare to duplicate the `llvm.threadlocal.address` intrinsic as necessary. Introduces a new `TargetLowering::addressingModeSupportsTLS` callback that allows targets to indicate whether TLS accesses can be part of an addressing mode. This is fixing a performance problem, as this folding of TLS-accesses into multiple addressing modes happened naturally before the introduction of the `llvm.threadlocal.address` intrinsic, but regressed due to `SelectionDAG` keeping things in registers when accessed across basic blocks, so CodeGenPrepare needs to duplicate to mitigate this. We see a ~0.5% recovery in a codebase with heavy TLS usage (HHVM). This fixes most of #87437 --- llvm/include/llvm/CodeGen/TargetLowering.h | 6 + llvm/lib/CodeGen/CodeGenPrepare.cpp | 29 ++- llvm/lib/Target/X86/X86ISelLowering.cpp | 24 +++ llvm/lib/Target/X86/X86ISelLowering.h | 2 + .../X86/codegen-prepare-addrmode-tls.ll | 193 ++++++++++++++++++ .../CodeGenPrepare/X86/sink-addrmode-tls.ll | 86 ++++++++ 6 files changed, 336 insertions(+), 4 deletions(-) create mode 100644 llvm/test/CodeGen/X86/codegen-prepare-addrmode-tls.ll create mode 100644 llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-tls.ll diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index e0ade0295902..2dd978c7b584 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -2805,6 +2805,12 @@ public: Type *Ty, unsigned AddrSpace, Instruction *I = nullptr) const; + /// Returns true if the targets addressing mode can target thread local + /// storage (TLS). + virtual bool addressingModeSupportsTLS(const GlobalValue &) const { + return false; + } + /// Return the prefered common base offset. virtual int64_t getPreferredLargeGEPBaseOffset(int64_t MinOffset, int64_t MaxOffset) const { diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp index e657872c3828..22a766f8d625 100644 --- a/llvm/lib/CodeGen/CodeGenPrepare.cpp +++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp @@ -5082,6 +5082,15 @@ bool AddressingModeMatcher::matchOperationAddr(User *AddrInst, unsigned Opcode, } return true; } + case Instruction::Call: + if (IntrinsicInst *II = dyn_cast(AddrInst)) { + if (II->getIntrinsicID() == Intrinsic::threadlocal_address) { + GlobalValue &GV = cast(*II->getArgOperand(0)); + if (TLI.addressingModeSupportsTLS(GV)) + return matchAddr(AddrInst->getOperand(0), Depth); + } + } + break; } return false; } @@ -5620,11 +5629,16 @@ bool CodeGenPrepare::optimizeMemoryInst(Instruction *MemoryInst, Value *Addr, return Modified; } - if (AddrMode.BaseGV) { + GlobalValue *BaseGV = AddrMode.BaseGV; + if (BaseGV != nullptr) { if (ResultPtr) return Modified; - ResultPtr = AddrMode.BaseGV; + if (BaseGV->isThreadLocal()) { + ResultPtr = Builder.CreateThreadLocalAddress(BaseGV); + } else { + ResultPtr = BaseGV; + } } // If the real base value actually came from an inttoptr, then the matcher @@ -5789,8 +5803,15 @@ bool CodeGenPrepare::optimizeMemoryInst(Instruction *MemoryInst, Value *Addr, } // Add in the BaseGV if present. - if (AddrMode.BaseGV) { - Value *V = Builder.CreatePtrToInt(AddrMode.BaseGV, IntPtrTy, "sunkaddr"); + GlobalValue *BaseGV = AddrMode.BaseGV; + if (BaseGV != nullptr) { + Value *BaseGVPtr; + if (BaseGV->isThreadLocal()) { + BaseGVPtr = Builder.CreateThreadLocalAddress(BaseGV); + } else { + BaseGVPtr = BaseGV; + } + Value *V = Builder.CreatePtrToInt(BaseGVPtr, IntPtrTy, "sunkaddr"); if (Result) Result = Builder.CreateAdd(Result, V, "sunkaddr"); else diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 27107f554fcc..bedec0c8974a 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -18920,6 +18920,30 @@ X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const { llvm_unreachable("TLS not implemented for this target."); } +bool X86TargetLowering::addressingModeSupportsTLS(const GlobalValue &GV) const { + if (Subtarget.is64Bit() && Subtarget.isTargetELF()) { + const TargetMachine &TM = getTargetMachine(); + TLSModel::Model Model = TM.getTLSModel(&GV); + switch (Model) { + case TLSModel::LocalExec: + case TLSModel::InitialExec: + // We can include the %fs segment register in addressing modes. + return true; + case TLSModel::LocalDynamic: + case TLSModel::GeneralDynamic: + // These models do not result in %fs relative addresses unless + // TLS descriptior are used. + // + // Even in the case of TLS descriptors we currently have no way to model + // the difference between %fs access and the computations needed for the + // offset and returning `true` for TLS-desc currently duplicates both + // which is detrimental :-/ + return false; + } + } + return false; +} + /// Lower SRA_PARTS and friends, which return two i32 values /// and take a 2 x i32 value to shift plus a shift amount. /// TODO: Can this be moved to general expansion code? diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h index 0a1e8ca44273..e348ba6e8ac0 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.h +++ b/llvm/lib/Target/X86/X86ISelLowering.h @@ -1323,6 +1323,8 @@ namespace llvm { Type *Ty, unsigned AS, Instruction *I = nullptr) const override; + bool addressingModeSupportsTLS(const GlobalValue &GV) const override; + /// Return true if the specified immediate is legal /// icmp immediate, that is the target has icmp instructions which can /// compare a register against the immediate without having to materialize diff --git a/llvm/test/CodeGen/X86/codegen-prepare-addrmode-tls.ll b/llvm/test/CodeGen/X86/codegen-prepare-addrmode-tls.ll new file mode 100644 index 000000000000..0ca1da26fa89 --- /dev/null +++ b/llvm/test/CodeGen/X86/codegen-prepare-addrmode-tls.ll @@ -0,0 +1,193 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc -o - %s | FileCheck %s --check-prefix=NOPIC +; RUN: llc -o - %s -relocation-model=pic | FileCheck %s --check-prefix=PIC +; RUN: llc -o - %s -relocation-model=pic -enable-tlsdesc | FileCheck %s --check-prefix=TLSDESC + +target triple = "x86_64--linux-gnu" + +declare void @effect() +declare nonnull ptr @llvm.threadlocal.address.p0(ptr nonnull) + +@foo_local = dso_local thread_local(localexec) global i32 0, align 4 + +define i32 @func_local_tls(i32 %arg0, i64 %arg1) nounwind { +; NOPIC-LABEL: func_local_tls: +; NOPIC: # %bb.0: # %entry +; NOPIC-NEXT: pushq %rbp +; NOPIC-NEXT: pushq %rbx +; NOPIC-NEXT: pushq %rax +; NOPIC-NEXT: movl %fs:foo_local@TPOFF, %ebp +; NOPIC-NEXT: testl %edi, %edi +; NOPIC-NEXT: movl %ebp, %eax +; NOPIC-NEXT: jne .LBB0_2 +; NOPIC-NEXT: # %bb.1: # %if.then +; NOPIC-NEXT: movq %rsi, %rbx +; NOPIC-NEXT: callq effect@PLT +; NOPIC-NEXT: movl %fs:foo_local@TPOFF+168(,%rbx,4), %eax +; NOPIC-NEXT: .LBB0_2: # %if.end +; NOPIC-NEXT: addl %ebp, %eax +; NOPIC-NEXT: addq $8, %rsp +; NOPIC-NEXT: popq %rbx +; NOPIC-NEXT: popq %rbp +; NOPIC-NEXT: retq +; +; PIC-LABEL: func_local_tls: +; PIC: # %bb.0: # %entry +; PIC-NEXT: pushq %rbp +; PIC-NEXT: pushq %r14 +; PIC-NEXT: pushq %rbx +; PIC-NEXT: movl %fs:.Lfoo_local$local@TPOFF, %ebp +; PIC-NEXT: testl %edi, %edi +; PIC-NEXT: movl %ebp, %eax +; PIC-NEXT: jne .LBB0_2 +; PIC-NEXT: # %bb.1: # %if.then +; PIC-NEXT: movq %rsi, %rbx +; PIC-NEXT: movq %fs:0, %rax +; PIC-NEXT: leaq .Lfoo_local$local@TPOFF(%rax), %r14 +; PIC-NEXT: callq effect@PLT +; PIC-NEXT: movl 168(%r14,%rbx,4), %eax +; PIC-NEXT: .LBB0_2: # %if.end +; PIC-NEXT: addl %ebp, %eax +; PIC-NEXT: popq %rbx +; PIC-NEXT: popq %r14 +; PIC-NEXT: popq %rbp +; PIC-NEXT: retq +; +; TLSDESC-LABEL: func_local_tls: +; TLSDESC: # %bb.0: # %entry +; TLSDESC-NEXT: pushq %rbp +; TLSDESC-NEXT: pushq %r14 +; TLSDESC-NEXT: pushq %rbx +; TLSDESC-NEXT: movl %fs:.Lfoo_local$local@TPOFF, %ebp +; TLSDESC-NEXT: testl %edi, %edi +; TLSDESC-NEXT: movl %ebp, %eax +; TLSDESC-NEXT: jne .LBB0_2 +; TLSDESC-NEXT: # %bb.1: # %if.then +; TLSDESC-NEXT: movq %rsi, %rbx +; TLSDESC-NEXT: movq %fs:0, %rax +; TLSDESC-NEXT: leaq .Lfoo_local$local@TPOFF(%rax), %r14 +; TLSDESC-NEXT: callq effect@PLT +; TLSDESC-NEXT: movl 168(%r14,%rbx,4), %eax +; TLSDESC-NEXT: .LBB0_2: # %if.end +; TLSDESC-NEXT: addl %ebp, %eax +; TLSDESC-NEXT: popq %rbx +; TLSDESC-NEXT: popq %r14 +; TLSDESC-NEXT: popq %rbp +; TLSDESC-NEXT: retq +entry: + %addr = tail call ptr @llvm.threadlocal.address.p0(ptr @foo_local) + %load0 = load i32, ptr %addr, align 4 + %cond = icmp eq i32 %arg0, 0 + br i1 %cond, label %if.then, label %if.end + +if.then: + tail call void @effect() + %x = add i64 %arg1, 42 + %addr1 = getelementptr inbounds i32, ptr %addr, i64 %x + %load1 = load i32, ptr %addr1, align 4 + br label %if.end + +if.end: + %phi = phi i32 [ %load1, %if.then ], [ %load0, %entry ] + %ret = add i32 %phi, %load0 + ret i32 %ret +} + +@foo_nonlocal = thread_local global i32 0, align 4 + +define i32 @func_nonlocal_tls(i32 %arg0, i64 %arg1) nounwind { +; NOPIC-LABEL: func_nonlocal_tls: +; NOPIC: # %bb.0: # %entry +; NOPIC-NEXT: pushq %rbp +; NOPIC-NEXT: pushq %r14 +; NOPIC-NEXT: pushq %rbx +; NOPIC-NEXT: movq foo_nonlocal@GOTTPOFF(%rip), %r14 +; NOPIC-NEXT: movl %fs:(%r14), %ebp +; NOPIC-NEXT: testl %edi, %edi +; NOPIC-NEXT: movl %ebp, %eax +; NOPIC-NEXT: jne .LBB1_2 +; NOPIC-NEXT: # %bb.1: # %if.then +; NOPIC-NEXT: movq %rsi, %rbx +; NOPIC-NEXT: callq effect@PLT +; NOPIC-NEXT: movl %fs:168(%r14,%rbx,4), %eax +; NOPIC-NEXT: .LBB1_2: # %if.end +; NOPIC-NEXT: addl %ebp, %eax +; NOPIC-NEXT: popq %rbx +; NOPIC-NEXT: popq %r14 +; NOPIC-NEXT: popq %rbp +; NOPIC-NEXT: retq +; +; PIC-LABEL: func_nonlocal_tls: +; PIC: # %bb.0: # %entry +; PIC-NEXT: pushq %rbp +; PIC-NEXT: pushq %r15 +; PIC-NEXT: pushq %r14 +; PIC-NEXT: pushq %rbx +; PIC-NEXT: pushq %rax +; PIC-NEXT: movq %rsi, %rbx +; PIC-NEXT: movl %edi, %ebp +; PIC-NEXT: data16 +; PIC-NEXT: leaq foo_nonlocal@TLSGD(%rip), %rdi +; PIC-NEXT: data16 +; PIC-NEXT: data16 +; PIC-NEXT: rex64 +; PIC-NEXT: callq __tls_get_addr@PLT +; PIC-NEXT: movq %rax, %r14 +; PIC-NEXT: movl (%rax), %r15d +; PIC-NEXT: testl %ebp, %ebp +; PIC-NEXT: movl %r15d, %eax +; PIC-NEXT: jne .LBB1_2 +; PIC-NEXT: # %bb.1: # %if.then +; PIC-NEXT: callq effect@PLT +; PIC-NEXT: movl 168(%r14,%rbx,4), %eax +; PIC-NEXT: .LBB1_2: # %if.end +; PIC-NEXT: addl %r15d, %eax +; PIC-NEXT: addq $8, %rsp +; PIC-NEXT: popq %rbx +; PIC-NEXT: popq %r14 +; PIC-NEXT: popq %r15 +; PIC-NEXT: popq %rbp +; PIC-NEXT: retq +; +; TLSDESC-LABEL: func_nonlocal_tls: +; TLSDESC: # %bb.0: # %entry +; TLSDESC-NEXT: pushq %rbp +; TLSDESC-NEXT: pushq %r14 +; TLSDESC-NEXT: pushq %rbx +; TLSDESC-NEXT: leaq foo_nonlocal@tlsdesc(%rip), %rax +; TLSDESC-NEXT: callq *foo_nonlocal@tlscall(%rax) +; TLSDESC-NEXT: movl %fs:(%rax), %ebp +; TLSDESC-NEXT: testl %edi, %edi +; TLSDESC-NEXT: movl %ebp, %ecx +; TLSDESC-NEXT: jne .LBB1_2 +; TLSDESC-NEXT: # %bb.1: # %if.then +; TLSDESC-NEXT: movq %rsi, %rbx +; TLSDESC-NEXT: addq %fs:0, %rax +; TLSDESC-NEXT: movq %rax, %r14 +; TLSDESC-NEXT: callq effect@PLT +; TLSDESC-NEXT: movl 168(%r14,%rbx,4), %ecx +; TLSDESC-NEXT: .LBB1_2: # %if.end +; TLSDESC-NEXT: addl %ebp, %ecx +; TLSDESC-NEXT: movl %ecx, %eax +; TLSDESC-NEXT: popq %rbx +; TLSDESC-NEXT: popq %r14 +; TLSDESC-NEXT: popq %rbp +; TLSDESC-NEXT: retq +entry: + %addr = tail call ptr @llvm.threadlocal.address.p0(ptr @foo_nonlocal) + %load0 = load i32, ptr %addr, align 4 + %cond = icmp eq i32 %arg0, 0 + br i1 %cond, label %if.then, label %if.end + +if.then: + tail call void @effect() + %x = add i64 %arg1, 42 + %addr1 = getelementptr inbounds i32, ptr %addr, i64 %x + %load1 = load i32, ptr %addr1, align 4 + br label %if.end + +if.end: + %phi = phi i32 [ %load1, %if.then ], [ %load0, %entry ] + %ret = add i32 %phi, %load0 + ret i32 %ret +} diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-tls.ll b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-tls.ll new file mode 100644 index 000000000000..080c807cbad1 --- /dev/null +++ b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-tls.ll @@ -0,0 +1,86 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S -passes='require,function(codegenprepare)' %s | FileCheck %s + +target triple = "x86_64--linux-gnu" + +@foo = dso_local thread_local(localexec) global i32 0, align 4 + +declare void @effect() +declare nonnull ptr @llvm.threadlocal.address.p0(ptr nonnull) + +define i32 @func0(i32 %arg) { +; CHECK-LABEL: define i32 @func0( +; CHECK-SAME: i32 [[ARG:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[ADDR:%.*]] = tail call ptr @llvm.threadlocal.address.p0(ptr @foo) +; CHECK-NEXT: [[LOAD0:%.*]] = load i32, ptr [[ADDR]], align 4 +; CHECK-NEXT: [[COND:%.*]] = icmp eq i32 [[ARG]], 0 +; CHECK-NEXT: br i1 [[COND]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] +; CHECK: if.then: +; CHECK-NEXT: tail call void @effect() +; CHECK-NEXT: [[TMP0:%.*]] = call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @foo) +; CHECK-NEXT: [[LOAD1:%.*]] = load i32, ptr [[TMP0]], align 4 +; CHECK-NEXT: br label [[IF_END]] +; CHECK: if.end: +; CHECK-NEXT: [[PHI:%.*]] = phi i32 [ [[LOAD1]], [[IF_THEN]] ], [ [[LOAD0]], [[ENTRY:%.*]] ] +; CHECK-NEXT: [[RET:%.*]] = add i32 [[PHI]], [[LOAD0]] +; CHECK-NEXT: ret i32 [[RET]] +; +entry: + %addr = tail call ptr @llvm.threadlocal.address.p0(ptr @foo) + %load0 = load i32, ptr %addr, align 4 + %cond = icmp eq i32 %arg, 0 + br i1 %cond, label %if.then, label %if.end + +if.then: + tail call void @effect() + %load1 = load i32, ptr %addr, align 4 + br label %if.end + +if.end: + %phi = phi i32 [ %load1, %if.then ], [ %load0, %entry ] + %ret = add i32 %phi, %load0 + ret i32 %ret +} + +define i32 @func1(i32 %arg0, i32 %arg1) { +; CHECK-LABEL: define i32 @func1( +; CHECK-SAME: i32 [[ARG0:%.*]], i32 [[ARG1:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[ADDR:%.*]] = tail call ptr @llvm.threadlocal.address.p0(ptr @foo) +; CHECK-NEXT: [[LOAD0:%.*]] = load i32, ptr [[ADDR]], align 4 +; CHECK-NEXT: [[COND:%.*]] = icmp eq i32 [[ARG0]], 0 +; CHECK-NEXT: br i1 [[COND]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] +; CHECK: if.then: +; CHECK-NEXT: tail call void @effect() +; CHECK-NEXT: [[X:%.*]] = add i32 [[ARG1]], 42 +; CHECK-NEXT: [[X64:%.*]] = sext i32 [[X]] to i64 +; CHECK-NEXT: [[TMP0:%.*]] = call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @foo) +; CHECK-NEXT: [[SUNKADDR:%.*]] = mul i64 [[X64]], 4 +; CHECK-NEXT: [[ADDR1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 [[SUNKADDR]] +; CHECK-NEXT: [[LOAD1:%.*]] = load i32, ptr [[ADDR1]], align 4 +; CHECK-NEXT: br label [[IF_END]] +; CHECK: if.end: +; CHECK-NEXT: [[PHI:%.*]] = phi i32 [ [[LOAD1]], [[IF_THEN]] ], [ [[LOAD0]], [[ENTRY:%.*]] ] +; CHECK-NEXT: [[RET:%.*]] = add i32 [[PHI]], [[LOAD0]] +; CHECK-NEXT: ret i32 [[RET]] +; +entry: + %addr = tail call ptr @llvm.threadlocal.address.p0(ptr @foo) + %load0 = load i32, ptr %addr, align 4 + %cond = icmp eq i32 %arg0, 0 + br i1 %cond, label %if.then, label %if.end + +if.then: + tail call void @effect() + %x = add i32 %arg1, 42 + %x64 = sext i32 %x to i64 + %addr1 = getelementptr inbounds i32, ptr %addr, i64 %x64 + %load1 = load i32, ptr %addr1, align 4 + br label %if.end + +if.end: + %phi = phi i32 [ %load1, %if.then ], [ %load0, %entry ] + %ret = add i32 %phi, %load0 + ret i32 %ret +} -- GitLab From 2583b2eea4a509424e5e5f51dffedd9beede76a3 Mon Sep 17 00:00:00 2001 From: Tom Eccles Date: Wed, 17 Apr 2024 19:46:43 +0000 Subject: [PATCH 099/862] [flang][NFC] Add missing include for FreeBSD Suggested by dankm here: https://github.com/llvm/llvm-project/pull/88517 --- flang/runtime/extensions.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/flang/runtime/extensions.cpp b/flang/runtime/extensions.cpp index 12498b502ae1..80604d886a70 100644 --- a/flang/runtime/extensions.cpp +++ b/flang/runtime/extensions.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #ifdef _WIN32 inline void CtimeBuffer(char *buffer, size_t bufsize, const time_t cur_time, -- GitLab From d0c51f7d5496015ce410cc758a7caf976ffaaec7 Mon Sep 17 00:00:00 2001 From: Kai Nacke Date: Wed, 17 Apr 2024 16:07:35 -0400 Subject: [PATCH 100/862] [LiveIns] Improve recomputeLiveIns() (#88951) Some small changes to recomputeLiveIns() to improve performance: - Instead of copying the list of old live-ins, and then clearing them, a new method swaps the list for an empty one. - getLiveIns() now returns a constant reference to the list As result, the list-data is never copied. Depending on the implementation details of the vector container, it can also save calls to allocate and deallocate memory. I see a small improvement on CTMark with these changes. --------- Co-authored-by: Nikita Popov --- llvm/include/llvm/CodeGen/LivePhysRegs.h | 9 +++++---- llvm/include/llvm/CodeGen/MachineBasicBlock.h | 6 +++++- llvm/lib/CodeGen/MachineBasicBlock.cpp | 6 ++++++ 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/llvm/include/llvm/CodeGen/LivePhysRegs.h b/llvm/include/llvm/CodeGen/LivePhysRegs.h index 9574a6f0c7c0..d315e4ff6f3a 100644 --- a/llvm/include/llvm/CodeGen/LivePhysRegs.h +++ b/llvm/include/llvm/CodeGen/LivePhysRegs.h @@ -199,14 +199,15 @@ void computeAndAddLiveIns(LivePhysRegs &LiveRegs, /// any changes were made. static inline bool recomputeLiveIns(MachineBasicBlock &MBB) { LivePhysRegs LPR; - auto oldLiveIns = MBB.getLiveIns(); + std::vector OldLiveIns; - MBB.clearLiveIns(); + MBB.clearLiveIns(OldLiveIns); computeAndAddLiveIns(LPR, MBB); MBB.sortUniqueLiveIns(); - auto newLiveIns = MBB.getLiveIns(); - return oldLiveIns != newLiveIns; + const std::vector &NewLiveIns = + MBB.getLiveIns(); + return OldLiveIns != NewLiveIns; } /// Convenience function for recomputing live-in's for a set of MBBs until the diff --git a/llvm/include/llvm/CodeGen/MachineBasicBlock.h b/llvm/include/llvm/CodeGen/MachineBasicBlock.h index dc2035fa598c..5b6be3a96b2f 100644 --- a/llvm/include/llvm/CodeGen/MachineBasicBlock.h +++ b/llvm/include/llvm/CodeGen/MachineBasicBlock.h @@ -441,6 +441,10 @@ public: /// Clear live in list. void clearLiveIns(); + /// Clear the live in list, and return the removed live in's in \p OldLiveIns. + /// Requires that the vector \p OldLiveIns is empty. + void clearLiveIns(std::vector &OldLiveIns); + /// Add PhysReg as live in to this block, and ensure that there is a copy of /// PhysReg to a virtual register of class RC. Return the virtual register /// that is a copy of the live in PhysReg. @@ -477,7 +481,7 @@ public: /// Remove entry from the livein set and return iterator to the next. livein_iterator removeLiveIn(livein_iterator I); - std::vector getLiveIns() const { return LiveIns; } + const std::vector &getLiveIns() const { return LiveIns; } class liveout_iterator { public: diff --git a/llvm/lib/CodeGen/MachineBasicBlock.cpp b/llvm/lib/CodeGen/MachineBasicBlock.cpp index b2114c250ac0..0bd5f09564ec 100644 --- a/llvm/lib/CodeGen/MachineBasicBlock.cpp +++ b/llvm/lib/CodeGen/MachineBasicBlock.cpp @@ -1728,6 +1728,12 @@ void MachineBasicBlock::clearLiveIns() { LiveIns.clear(); } +void MachineBasicBlock::clearLiveIns( + std::vector &OldLiveIns) { + assert(OldLiveIns.empty() && "Vector must be empty"); + std::swap(LiveIns, OldLiveIns); +} + MachineBasicBlock::livein_iterator MachineBasicBlock::livein_begin() const { assert(getParent()->getProperties().hasProperty( MachineFunctionProperties::Property::TracksLiveness) && -- GitLab From 60b90b523323f8196a9e4a68b1f33358624c09eb Mon Sep 17 00:00:00 2001 From: Zequan Wu Date: Wed, 17 Apr 2024 16:09:38 -0400 Subject: [PATCH 101/862] [lldb][DynamicLoader] Fix lldb unable to stop at _dl_debug_state if user set it before the process launched. (#88792) If user sets a breakpoint at `_dl_debug_state` before the process launched, the breakpoint is not resolved yet. When lldb loads dynamic loader module, it's created with `Target::GetOrCreateModule` which notifies any pending breakpoint to resolve. However, the module's sections are not loaded at this time. They are loaded after returned from [Target::GetOrCreateModule](https://github.com/llvm/llvm-project/blob/0287a5cc4e2a5ded1ae2e4079f91052e6a6b8d9b/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DynamicLoaderPOSIXDYLD.cpp#L574-L577). This change fixes it by manually resolving breakpoints after creating dynamic loader module. --- .../POSIX-DYLD/DynamicLoaderPOSIXDYLD.cpp | 11 +++++++++-- .../breakpoint_command/TestBreakpointCommand.py | 17 +++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DynamicLoaderPOSIXDYLD.cpp b/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DynamicLoaderPOSIXDYLD.cpp index 9baf86da4dc7..9fa245fc41d4 100644 --- a/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DynamicLoaderPOSIXDYLD.cpp +++ b/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DynamicLoaderPOSIXDYLD.cpp @@ -571,10 +571,17 @@ ModuleSP DynamicLoaderPOSIXDYLD::LoadInterpreterModule() { FileSpec file(info.GetName().GetCString()); ModuleSpec module_spec(file, target.GetArchitecture()); - if (ModuleSP module_sp = target.GetOrCreateModule(module_spec, - true /* notify */)) { + // Don't notify that module is added here because its loading section + // addresses are not updated yet. We manually notify it below. + if (ModuleSP module_sp = + target.GetOrCreateModule(module_spec, /*notify=*/false)) { UpdateLoadedSections(module_sp, LLDB_INVALID_ADDRESS, m_interpreter_base, false); + // Manually notify that dynamic linker is loaded after updating load section + // addersses so that breakpoints can be resolved. + ModuleList module_list; + module_list.Append(module_sp); + target.ModulesDidLoad(module_list); m_interpreter_module = module_sp; return module_sp; } diff --git a/lldb/test/API/functionalities/breakpoint/breakpoint_command/TestBreakpointCommand.py b/lldb/test/API/functionalities/breakpoint/breakpoint_command/TestBreakpointCommand.py index ea242952e409..850235fdcefa 100644 --- a/lldb/test/API/functionalities/breakpoint/breakpoint_command/TestBreakpointCommand.py +++ b/lldb/test/API/functionalities/breakpoint/breakpoint_command/TestBreakpointCommand.py @@ -671,3 +671,20 @@ class BreakpointCommandTestCase(TestBase): self.assertNotEqual(breakpoints_stats, None) for breakpoint_stats in breakpoints_stats: self.assertIn("hitCount", breakpoint_stats) + + @skipIf(oslist=no_match(["linux"])) + def test_break_at__dl_debug_state(self): + """ + Test lldb is able to stop at _dl_debug_state if it is set before the + process is launched. + """ + self.build() + exe = self.getBuildArtifact("a.out") + self.runCmd("target create %s" % exe) + bpid = lldbutil.run_break_set_by_symbol( + self, "_dl_debug_state", num_expected_locations=0 + ) + self.runCmd("run") + self.assertIsNotNone( + lldbutil.get_one_thread_stopped_at_breakpoint_id(self.process(), bpid) + ) -- GitLab From e15f47f2675a5400464aec00219658882df5e3fa Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Wed, 17 Apr 2024 13:16:08 -0700 Subject: [PATCH 102/862] [InstCombine] Don't use dominating conditions to transform sub into xor. (#88566) Other passes are unable to reverse this transform if we use dominating conditions. Fixes #88239. --- llvm/include/llvm/Analysis/SimplifyQuery.h | 6 ++++++ .../Transforms/InstCombine/InstCombineAddSub.cpp | 6 ++++-- llvm/test/Transforms/InstCombine/sub-xor.ll | 4 +++- llvm/test/Transforms/PhaseOrdering/X86/pr88239.ll | 13 ++++++------- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/llvm/include/llvm/Analysis/SimplifyQuery.h b/llvm/include/llvm/Analysis/SimplifyQuery.h index e5e6ae0d3d8e..a10c0dc49fa2 100644 --- a/llvm/include/llvm/Analysis/SimplifyQuery.h +++ b/llvm/include/llvm/Analysis/SimplifyQuery.h @@ -113,6 +113,12 @@ struct SimplifyQuery { using namespace PatternMatch; return match(V, m_Undef()); } + + SimplifyQuery getWithoutDomCondCache() const { + SimplifyQuery Copy(*this); + Copy.DC = nullptr; + return Copy; + } }; } // end namespace llvm diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp index c59b867b10e7..df7f02810ee4 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp @@ -2281,8 +2281,10 @@ Instruction *InstCombinerImpl::visitSub(BinaryOperator &I) { if (match(Op0, m_APInt(Op0C))) { if (Op0C->isMask()) { // Turn this into a xor if LHS is 2^n-1 and the remaining bits are known - // zero. - KnownBits RHSKnown = computeKnownBits(Op1, 0, &I); + // zero. We don't use information from dominating conditions so this + // transform is easier to reverse if necessary. + KnownBits RHSKnown = llvm::computeKnownBits( + Op1, 0, SQ.getWithInstruction(&I).getWithoutDomCondCache()); if ((*Op0C | RHSKnown.Zero).isAllOnes()) return BinaryOperator::CreateXor(Op1, Op0); } diff --git a/llvm/test/Transforms/InstCombine/sub-xor.ll b/llvm/test/Transforms/InstCombine/sub-xor.ll index b4e87d0405fc..2976598e043f 100644 --- a/llvm/test/Transforms/InstCombine/sub-xor.ll +++ b/llvm/test/Transforms/InstCombine/sub-xor.ll @@ -158,13 +158,15 @@ define <2 x i8> @xor_add_splat_undef(<2 x i8> %x) { ret <2 x i8> %add } +; Make sure we don't convert sub to xor using dominating condition. That makes +; it hard for other passe to reverse. define i32 @xor_dominating_cond(i32 %x) { ; CHECK-LABEL: @xor_dominating_cond( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[COND:%.*]] = icmp ult i32 [[X:%.*]], 256 ; CHECK-NEXT: br i1 [[COND]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] ; CHECK: if.then: -; CHECK-NEXT: [[A:%.*]] = xor i32 [[X]], 255 +; CHECK-NEXT: [[A:%.*]] = sub nuw nsw i32 255, [[X]] ; CHECK-NEXT: ret i32 [[A]] ; CHECK: if.end: ; CHECK-NEXT: ret i32 [[X]] diff --git a/llvm/test/Transforms/PhaseOrdering/X86/pr88239.ll b/llvm/test/Transforms/PhaseOrdering/X86/pr88239.ll index 3afa1904fb24..1c16b37144f1 100644 --- a/llvm/test/Transforms/PhaseOrdering/X86/pr88239.ll +++ b/llvm/test/Transforms/PhaseOrdering/X86/pr88239.ll @@ -8,19 +8,18 @@ define void @foo(ptr noalias noundef %0, ptr noalias noundef %1) optsize { ; CHECK-LABEL: define void @foo( ; CHECK-SAME: ptr noalias nocapture noundef readonly [[TMP0:%.*]], ptr noalias nocapture noundef writeonly [[TMP1:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] { ; CHECK-NEXT: vector.ph: +; CHECK-NEXT: [[INVARIANT_GEP:%.*]] = getelementptr i8, ptr [[TMP0]], i64 -28 ; CHECK-NEXT: br label [[TMP4:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[TMP2:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[TMP4]] ] -; CHECK-NEXT: [[VEC_IND:%.*]] = phi <8 x i64> [ , [[TMP2]] ], [ [[VEC_IND_NEXT:%.*]], [[TMP4]] ] -; CHECK-NEXT: [[TMP6:%.*]] = and <8 x i64> [[VEC_IND]], -; CHECK-NEXT: [[TMP3:%.*]] = xor <8 x i64> [[TMP6]], -; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], <8 x i64> [[TMP3]] -; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = tail call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> [[TMP7]], i32 4, <8 x i1> , <8 x i32> poison) +; CHECK-NEXT: [[TMP3:%.*]] = sub nuw nsw i64 255, [[INDVARS_IV]] +; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[INVARIANT_GEP]], i64 [[TMP3]] +; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = load <8 x i32>, ptr [[GEP]], align 4 ; CHECK-NEXT: [[TMP5:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_GATHER]], +; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <8 x i32> [[TMP5]], <8 x i32> poison, <8 x i32> ; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 [[INDVARS_IV]] -; CHECK-NEXT: store <8 x i32> [[TMP5]], ptr [[TMP10]], align 4 +; CHECK-NEXT: store <8 x i32> [[TMP6]], ptr [[TMP10]], align 4 ; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw i64 [[INDVARS_IV]], 8 -; CHECK-NEXT: [[VEC_IND_NEXT]] = add <8 x i64> [[VEC_IND]], ; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 256 ; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[MIDDLE_BLOCK:%.*]], label [[TMP4]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: -- GitLab From f309c882ea5ad411ac1011a4228c0e49b1025bb4 Mon Sep 17 00:00:00 2001 From: Tom Eccles Date: Wed, 17 Apr 2024 20:11:35 +0000 Subject: [PATCH 103/862] Revert "[flang][NFC] Add missing include for FreeBSD" This reverts commit 2583b2eea4a509424e5e5f51dffedd9beede76a3. --- flang/runtime/extensions.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/flang/runtime/extensions.cpp b/flang/runtime/extensions.cpp index 80604d886a70..12498b502ae1 100644 --- a/flang/runtime/extensions.cpp +++ b/flang/runtime/extensions.cpp @@ -21,7 +21,6 @@ #include #include #include -#include #ifdef _WIN32 inline void CtimeBuffer(char *buffer, size_t bufsize, const time_t cur_time, -- GitLab From 0cee89431d77d0bb0809fd9b2c9d21da2a2783aa Mon Sep 17 00:00:00 2001 From: Tom Eccles Date: Wed, 17 Apr 2024 19:46:43 +0000 Subject: [PATCH 104/862] [flang][NFC] Add missing include for FreeBSD Suggested by dankm here: https://github.com/llvm/llvm-project/pull/88517 --- flang/runtime/extensions.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flang/runtime/extensions.cpp b/flang/runtime/extensions.cpp index 12498b502ae1..4b110cc10c84 100644 --- a/flang/runtime/extensions.cpp +++ b/flang/runtime/extensions.cpp @@ -43,9 +43,9 @@ inline void CtimeBuffer(char *buffer, size_t bufsize, const time_t cur_time, } #endif -#if _REENTRANT || _POSIX_C_SOURCE >= 199506L -// System is posix-compliant and has getlogin_r -#include +#ifndef _WIN32 +// posix-compliant and has getlogin_r and F_OK +#include #endif extern "C" { -- GitLab From 9435edf628cb2011652897e1f10f7c55313d50be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?= =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?= =?UTF-8?q?=E3=83=B3=29?= Date: Wed, 17 Apr 2024 13:45:22 -0700 Subject: [PATCH 105/862] [flang][cuda] Lower DEALLOCATE for device variables (#89091) Replace the runtime call to `AllocatableDeallocate` for CUDA device variable to the newly added `fir.cuda_deallocate` operation. This is similar with #88980 A third patch will handle the case of automatic dealloctaion of device allocatable variables --- flang/lib/Lower/Allocatable.cpp | 33 ++++++++++++++++++++-- flang/test/Lower/CUDA/cuda-allocatable.cuf | 15 ++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/flang/lib/Lower/Allocatable.cpp b/flang/lib/Lower/Allocatable.cpp index 1d434d512d0c..38f61528d7e2 100644 --- a/flang/lib/Lower/Allocatable.cpp +++ b/flang/lib/Lower/Allocatable.cpp @@ -799,6 +799,28 @@ static void postDeallocationAction(Fortran::lower::AbstractConverter &converter, Fortran::lower::attachDeclarePostDeallocAction(converter, builder, sym); } +static mlir::Value genCudaDeallocate(fir::FirOpBuilder &builder, + mlir::Location loc, + const fir::MutableBoxValue &box, + ErrorManager &errorManager, + const Fortran::semantics::Symbol &sym) { + fir::CUDADataAttributeAttr cudaAttr = + Fortran::lower::translateSymbolCUDADataAttribute(builder.getContext(), + sym); + mlir::Value errmsg = + mlir::isa(errorManager.errMsgAddr.getDefiningOp()) + ? nullptr + : errorManager.errMsgAddr; + + // Keep return type the same as a standard AllocatableAllocate call. + mlir::Type retTy = fir::runtime::getModel()(builder.getContext()); + return builder + .create( + loc, retTy, box.getAddr(), errmsg, cudaAttr, + errorManager.hasStatSpec() ? builder.getUnitAttr() : nullptr) + .getResult(); +} + // Generate deallocation of a pointer/allocatable. static mlir::Value genDeallocate(fir::FirOpBuilder &builder, @@ -806,10 +828,11 @@ genDeallocate(fir::FirOpBuilder &builder, const fir::MutableBoxValue &box, ErrorManager &errorManager, mlir::Value declaredTypeDesc = {}, const Fortran::semantics::Symbol *symbol = nullptr) { + bool isCudaSymbol = symbol && Fortran::semantics::HasCUDAAttr(*symbol); // Deallocate intrinsic types inline. if (!box.isDerived() && !box.isPolymorphic() && !box.isUnlimitedPolymorphic() && !errorManager.hasStatSpec() && - !useAllocateRuntime && !box.isPointer()) { + !useAllocateRuntime && !box.isPointer() && !isCudaSymbol) { // Pointers must use PointerDeallocate so that their deallocations // can be validated. mlir::Value ret = fir::factory::genFreemem(builder, loc, box); @@ -820,8 +843,12 @@ genDeallocate(fir::FirOpBuilder &builder, // Use runtime calls to deallocate descriptor cases. Sync MutableBoxValue // with its descriptor before and after calls if needed. errorManager.genStatCheck(builder, loc); - mlir::Value stat = - genRuntimeDeallocate(builder, loc, box, errorManager, declaredTypeDesc); + mlir::Value stat; + if (!isCudaSymbol) + stat = + genRuntimeDeallocate(builder, loc, box, errorManager, declaredTypeDesc); + else + stat = genCudaDeallocate(builder, loc, box, errorManager, *symbol); fir::factory::syncMutableBoxFromIRBox(builder, loc, box); if (symbol) postDeallocationAction(converter, builder, *symbol); diff --git a/flang/test/Lower/CUDA/cuda-allocatable.cuf b/flang/test/Lower/CUDA/cuda-allocatable.cuf index 55223011e8d9..5b10334ecdbc 100644 --- a/flang/test/Lower/CUDA/cuda-allocatable.cuf +++ b/flang/test/Lower/CUDA/cuda-allocatable.cuf @@ -5,6 +5,8 @@ subroutine sub1() real, allocatable, device :: a(:) allocate(a(10)) + + deallocate(a) end subroutine ! CHECK-LABEL: func.func @_QPsub1() @@ -13,10 +15,14 @@ end subroutine ! CHECK: fir.call @_FortranAAllocatableSetBounds ! CHECK: %{{.*}} = fir.cuda_allocate %[[BOX_DECL]]#1 : !fir.ref>>> {cuda_attr = #fir.cuda} -> i32 +! CHECK: %{{.*}} = fir.cuda_deallocate %[[BOX_DECL]]#1 : !fir.ref>>> {cuda_attr = #fir.cuda} -> i32 + subroutine sub2() real, allocatable, managed :: a(:) integer :: istat allocate(a(10), stat=istat) + + deallocate(a, stat=istat) end subroutine ! CHECK-LABEL: func.func @_QPsub2() @@ -28,6 +34,9 @@ end subroutine ! CHECK: %[[STAT:.*]] = fir.cuda_allocate %[[BOX_DECL]]#1 : !fir.ref>>> {cuda_attr = #fir.cuda, hasStat} -> i32 ! CHECK: fir.store %[[STAT]] to %[[ISTAT_DECL]]#1 : !fir.ref +! CHECK: %[[STAT:.*]] = fir.cuda_deallocate %[[BOX_DECL]]#1 : !fir.ref>>> {cuda_attr = #fir.cuda, hasStat} -> i32 +! CHECK: fir.store %[[STAT]] to %[[ISTAT_DECL]]#1 : !fir.ref + subroutine sub3() integer, allocatable, pinned :: a(:,:) logical :: plog @@ -92,6 +101,8 @@ subroutine sub7() integer :: istat character(50) :: err allocate(a(100), stat=istat, errmsg=err) + + deallocate(a, stat=istat, errmsg=err) end subroutine ! CHECK-LABEL: func.func @_QPsub7() @@ -105,3 +116,7 @@ end subroutine ! CHECK: fir.call @_FortranAAllocatableSetBounds ! CHECK: %[[STAT:.*]] = fir.cuda_allocate %[[BOX_DECL]]#1 : !fir.ref>>> errmsg(%[[ERR_BOX]] : !fir.box>) {cuda_attr = #fir.cuda, hasStat} -> i32 ! CHECK: fir.store %[[STAT]] to %[[ISTAT_DECL]]#1 : !fir.ref + +! CHECK: %[[ERR_BOX:.*]] = fir.embox %[[ERR_DECL]]#1 : (!fir.ref>) -> !fir.box> +! CHECK: %[[STAT:.*]] = fir.cuda_deallocate %[[BOX_DECL]]#1 : !fir.ref>>> errmsg(%15 : !fir.box>) {cuda_attr = #fir.cuda, hasStat} -> i32 +! CHECK: fir.store %[[STAT]] to %[[ISTAT_DECL]]#1 : !fir.ref -- GitLab From 1b87418b9960983f54aa67f87018466ae12d0ec6 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Wed, 17 Apr 2024 17:03:39 -0400 Subject: [PATCH 106/862] [bazel] Improve liblldb building (#89095) On Linux using --version-script doesn't force loading of the underlying archives that contain the symbols. By setting alwayslink=True on the API cc_library we virtually get this behavior. This also allows downstream users to use the exports files used by cmake. We could build more configurability into this but there are also a lot of possible variations users might want. --- .../llvm-project-overlay/lldb/BUILD.bazel | 42 +++++++++++-------- .../lldb/source/Plugins/BUILD.bazel | 20 +-------- 2 files changed, 26 insertions(+), 36 deletions(-) diff --git a/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel b/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel index 1f2b5b476bcc..7958c6024875 100644 --- a/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel @@ -16,7 +16,11 @@ package( licenses(["notice"]) -exports_files(["LICENSE.TXT"]) +exports_files([ + "LICENSE.TXT", + "source/API/liblldb-private.exports", + "source/API/liblldb.exports", +]) bool_flag( name = "enable_curses", @@ -203,13 +207,29 @@ cc_library( ":TargetHeaders", ":Utility", ":Version", + "//lldb/source/Plugins:PluginExpressionParserClang", "//lldb/source/Plugins:PluginExpressionParserClangHeaders", "//lldb/source/Plugins:PluginsConfig", + "//llvm:AllTargetsDisassemblers", "//llvm:ExecutionEngine", "//llvm:MCJIT", "//llvm:Support", "//llvm:config", - ], + ] + [ + "//lldb/source/Plugins:Plugin{}".format(x) + for x in DEFAULT_PLUGINS + DEFAULT_SCRIPT_PLUGINS + ] + select({ + "@platforms//os:macos": [ + "//lldb/source/Plugins:PluginProcessMacOSXKernel", + "//lldb/source/Plugins:PluginSymbolLocatorDebugSymbols", + "//lldb/source/Plugins:PluginSymbolVendorMacOSX", + ], + "@platforms//os:linux": [ + "//lldb/source/Plugins:PluginProcessLinux", + ], + "//conditions:default": [], + }), + alwayslink = True, ) cc_library( @@ -690,23 +710,8 @@ cc_library( name = "liblldb.static", deps = [ ":API", - ":Host", ":Interpreter", - "//llvm:AllTargetsDisassemblers", - ] + [ - "//lldb/source/Plugins:Plugin{}".format(x) - for x in DEFAULT_PLUGINS + DEFAULT_SCRIPT_PLUGINS - ] + select({ - "@platforms//os:macos": [ - "//lldb/source/Plugins:PluginProcessMacOSXKernel", - "//lldb/source/Plugins:PluginSymbolLocatorDebugSymbols", - "//lldb/source/Plugins:PluginSymbolVendorMacOSX", - ], - "@platforms//os:linux": [ - "//lldb/source/Plugins:PluginProcessLinux", - ], - "//conditions:default": [], - }), + ], ) cc_shared_library( @@ -889,6 +894,7 @@ cc_binary( deps = [ ":Host", ":Initialization", + ":Interpreter", ":Utility", ":Version", ":lldb_server_opts_gen", diff --git a/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel b/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel index b5f5bed1698a..e0907a838148 100644 --- a/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel @@ -557,7 +557,6 @@ cc_library( "//lldb:Breakpoint", "//lldb:Core", "//lldb:Host", - "//lldb:Interpreter", "//lldb:InterpreterHeaders", "//lldb:Target", "//lldb:TargetHeaders", @@ -591,7 +590,6 @@ cc_library( "//lldb:Core", "//lldb:Expression", "//lldb:Host", - "//lldb:Interpreter", "//lldb:InterpreterHeaders", "//lldb:Target", "//lldb:TargetHeaders", @@ -664,7 +662,7 @@ cc_library( "//lldb:Breakpoint", "//lldb:Core", "//lldb:Host", - "//lldb:Interpreter", + "//lldb:InterpreterHeaders", "//lldb:SymbolHeaders", "//lldb:Target", "//lldb:TargetHeaders", @@ -1264,7 +1262,6 @@ cc_library( "//lldb:Breakpoint", "//lldb:Core", "//lldb:Host", - "//lldb:Interpreter", "//lldb:InterpreterHeaders", "//lldb:Symbol", "//lldb:SymbolHeaders", @@ -1422,7 +1419,6 @@ cc_library( deps = [ ":PluginProcessUtility", "//lldb:Core", - "//lldb:Interpreter", "//lldb:InterpreterHeaders", "//lldb:Symbol", "//lldb:SymbolHeaders", @@ -1438,7 +1434,6 @@ cc_library( deps = [ ":PluginProcessUtility", "//lldb:Core", - "//lldb:Interpreter", "//lldb:InterpreterHeaders", "//lldb:Symbol", "//lldb:SymbolHeaders", @@ -1475,7 +1470,6 @@ cc_library( ":PluginProcessUtility", "//lldb:Core", "//lldb:Host", - "//lldb:Interpreter", "//lldb:InterpreterHeaders", "//lldb:Symbol", "//lldb:SymbolHeaders", @@ -1495,7 +1489,6 @@ cc_library( ":PluginProcessUtility", "//lldb:Core", "//lldb:Host", - "//lldb:Interpreter", "//lldb:InterpreterHeaders", "//lldb:Symbol", "//lldb:SymbolHeaders", @@ -1515,7 +1508,6 @@ cc_library( deps = [ ":PluginProcessUtility", "//lldb:Core", - "//lldb:Interpreter", "//lldb:InterpreterHeaders", "//lldb:Symbol", "//lldb:SymbolHeaders", @@ -1532,7 +1524,6 @@ cc_library( deps = [ ":PluginProcessUtility", "//lldb:Core", - "//lldb:Interpreter", "//lldb:InterpreterHeaders", "//lldb:Symbol", "//lldb:SymbolHeaders", @@ -1570,7 +1561,6 @@ cc_library( "//lldb:Expression", "//lldb:Headers", "//lldb:Host", - "//lldb:Interpreter", "//lldb:InterpreterHeaders", "//lldb:Symbol", "//lldb:SymbolHeaders", @@ -1608,7 +1598,7 @@ cc_library( "//lldb:Core", "//lldb:Expression", "//lldb:Headers", - "//lldb:Interpreter", + "//lldb:InterpreterHeaders", "//lldb:Symbol", "//lldb:SymbolHeaders", "//lldb:Target", @@ -1629,7 +1619,6 @@ cc_library( "//lldb:Expression", "//lldb:Headers", "//lldb:Host", - "//lldb:Interpreter", "//lldb:InterpreterHeaders", "//lldb:Symbol", "//lldb:SymbolHeaders", @@ -1682,7 +1671,6 @@ cc_library( ":PluginObjectFileMachO", "//lldb:Breakpoint", "//lldb:Core", - "//lldb:Interpreter", "//lldb:InterpreterHeaders", "//lldb:Symbol", "//lldb:SymbolHeaders", @@ -1838,7 +1826,6 @@ cc_library( include_prefix = "Plugins", deps = [ "//lldb:Core", - "//lldb:Interpreter", "//lldb:InterpreterHeaders", "//lldb:Utility", "//llvm:Support", @@ -2215,7 +2202,6 @@ cc_library( "//lldb:Core", "//lldb:Headers", "//lldb:Host", - "//lldb:Interpreter", "//lldb:InterpreterHeaders", "//lldb:Symbol", "//lldb:SymbolHeaders", @@ -2262,7 +2248,6 @@ cc_library( "//lldb:DataFormattersHeaders", "//lldb:ExpressionHeaders", "//lldb:Headers", - "//lldb:Interpreter", "//lldb:InterpreterHeaders", "//lldb:Symbol", "//lldb:SymbolHeaders", @@ -2287,7 +2272,6 @@ cc_library( "//lldb:Expression", "//lldb:Headers", "//lldb:Host", - "//lldb:Interpreter", "//lldb:Symbol", "//lldb:Target", "//lldb:TargetHeaders", -- GitLab From 0ab3f160c4bff1c7d57c046b95ab8c5035ae986f Mon Sep 17 00:00:00 2001 From: Philip Reames Date: Wed, 17 Apr 2024 14:23:38 -0700 Subject: [PATCH 107/862] [RISCV] Add coverage of add (mul X, C), Y oppurtunity using shNadd --- llvm/test/CodeGen/RISCV/rv64zba.ll | 73 ++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/llvm/test/CodeGen/RISCV/rv64zba.ll b/llvm/test/CodeGen/RISCV/rv64zba.ll index 6939185947f4..ccb23fc2bbfa 100644 --- a/llvm/test/CodeGen/RISCV/rv64zba.ll +++ b/llvm/test/CodeGen/RISCV/rv64zba.ll @@ -479,6 +479,18 @@ define i64 @addmul20(i64 %a, i64 %b) { ret i64 %d } +define i64 @addmul22(i64 %a, i64 %b) { +; CHECK-LABEL: addmul22: +; CHECK: # %bb.0: +; CHECK-NEXT: li a2, 22 +; CHECK-NEXT: mul a0, a0, a2 +; CHECK-NEXT: add a0, a0, a1 +; CHECK-NEXT: ret + %c = mul i64 %a, 22 + %d = add i64 %c, %b + ret i64 %d +} + define i64 @addmul24(i64 %a, i64 %b) { ; RV64I-LABEL: addmul24: ; RV64I: # %bb.0: @@ -551,6 +563,67 @@ define i64 @addmul72(i64 %a, i64 %b) { ret i64 %d } +define i64 @addmul162(i64 %a, i64 %b) { +; CHECK-LABEL: addmul162: +; CHECK: # %bb.0: +; CHECK-NEXT: li a2, 162 +; CHECK-NEXT: mul a0, a0, a2 +; CHECK-NEXT: add a0, a0, a1 +; CHECK-NEXT: ret + %c = mul i64 %a, 162 + %d = add i64 %c, %b + ret i64 %d +} + +define i64 @addmul180(i64 %a, i64 %b) { +; CHECK-LABEL: addmul180: +; CHECK: # %bb.0: +; CHECK-NEXT: li a2, 180 +; CHECK-NEXT: mul a0, a0, a2 +; CHECK-NEXT: add a0, a0, a1 +; CHECK-NEXT: ret + %c = mul i64 %a, 180 + %d = add i64 %c, %b + ret i64 %d +} + +define i64 @add255mul180(i64 %a) { +; CHECK-LABEL: add255mul180: +; CHECK: # %bb.0: +; CHECK-NEXT: li a1, 180 +; CHECK-NEXT: mul a0, a0, a1 +; CHECK-NEXT: addi a0, a0, 255 +; CHECK-NEXT: ret + %c = mul i64 %a, 180 + %d = add i64 %c, 255 + ret i64 %d +} + + +define i64 @addmul4096(i64 %a, i64 %b) { +; CHECK-LABEL: addmul4096: +; CHECK: # %bb.0: +; CHECK-NEXT: slli a0, a0, 12 +; CHECK-NEXT: add a0, a0, a1 +; CHECK-NEXT: ret + %c = mul i64 %a, 4096 + %d = add i64 %c, %b + ret i64 %d +} + +define i64 @addmul4230(i64 %a, i64 %b) { +; CHECK-LABEL: addmul4230: +; CHECK: # %bb.0: +; CHECK-NEXT: lui a2, 1 +; CHECK-NEXT: addiw a2, a2, 134 +; CHECK-NEXT: mul a0, a0, a2 +; CHECK-NEXT: add a0, a0, a1 +; CHECK-NEXT: ret + %c = mul i64 %a, 4230 + %d = add i64 %c, %b + ret i64 %d +} + define i64 @mul96(i64 %a) { ; RV64I-LABEL: mul96: ; RV64I: # %bb.0: -- GitLab From 678f19f08296fec299438130cf5943714c590b7e Mon Sep 17 00:00:00 2001 From: azhan92 Date: Wed, 17 Apr 2024 17:56:42 -0400 Subject: [PATCH 108/862] [Support] Report EISDIR when opening a directory (#79880) The test `llvm/unittests/Support/CommandLineTest.cpp` that handles errors in expansion of response files was previously disabled for AIX. Originally the code was dependent on `read` returning `EISDIR` which occurs on platforms such as Linux. However, other platforms such as AIX allow use of `read` on file descriptors for directories. This change updates `readNativeFile` to produce `EISDIR` on AIX and z/OS when used on a directory (instead of relying on the call to `read` to do so). --------- Co-authored-by: Alison Zhang Co-authored-by: James Henderson <46713263+jh7370@users.noreply.github.com> --- llvm/lib/Support/Unix/Path.inc | 9 ++++++++ .../tools/llvm-symbolizer/input-file-err.test | 2 -- llvm/unittests/Support/CommandLineTest.cpp | 2 -- llvm/unittests/Support/Path.cpp | 22 +++++++++++++++++++ 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Support/Unix/Path.inc b/llvm/lib/Support/Unix/Path.inc index 968e2c459f3f..6e679f74869f 100644 --- a/llvm/lib/Support/Unix/Path.inc +++ b/llvm/lib/Support/Unix/Path.inc @@ -1191,6 +1191,15 @@ Expected readNativeFile(file_t FD, MutableArrayRef Buf) { ssize_t NumRead = sys::RetryAfterSignal(-1, ::read, FD, Buf.data(), Size); if (ssize_t(NumRead) == -1) return errorCodeToError(errnoAsErrorCode()); +// The underlying operation on these platforms allow opening directories +// for reading in more cases than other platforms. +#if defined(__MVS__) || defined(_AIX) + struct stat Status; + if (fstat(FD, &Status) == -1) + return errorCodeToError(errnoAsErrorCode()); + if (S_ISDIR(Status.st_mode)) + return errorCodeToError(make_error_code(errc::is_a_directory)); +#endif return NumRead; } diff --git a/llvm/test/tools/llvm-symbolizer/input-file-err.test b/llvm/test/tools/llvm-symbolizer/input-file-err.test index df14da2f433c..76115b513470 100644 --- a/llvm/test/tools/llvm-symbolizer/input-file-err.test +++ b/llvm/test/tools/llvm-symbolizer/input-file-err.test @@ -1,5 +1,3 @@ -Failing on AIX due to D153595. The test expects a different error message from the one given on AIX. -XFAIL: target={{.*}}-aix{{.*}} RUN: not llvm-addr2line -e %p/Inputs/nonexistent 0x12 2>&1 | FileCheck %s --check-prefix=CHECK-NONEXISTENT-A2L -DMSG=%errc_ENOENT RUN: not llvm-addr2line -e %p/Inputs/nonexistent 2>&1 | FileCheck %s --check-prefix=CHECK-NONEXISTENT-A2L -DMSG=%errc_ENOENT CHECK-NONEXISTENT-A2L: llvm-addr2line{{.*}}: error: '{{.*}}Inputs/nonexistent': [[MSG]] diff --git a/llvm/unittests/Support/CommandLineTest.cpp b/llvm/unittests/Support/CommandLineTest.cpp index bbeb9d5dc19b..23f6081cd32a 100644 --- a/llvm/unittests/Support/CommandLineTest.cpp +++ b/llvm/unittests/Support/CommandLineTest.cpp @@ -1117,7 +1117,6 @@ TEST(CommandLineTest, BadResponseFile) { ASSERT_STREQ(Argv[0], "clang"); ASSERT_STREQ(Argv[1], AFileExp.c_str()); -#if !defined(_AIX) && !defined(__MVS__) std::string ADirExp = std::string("@") + std::string(ADir.path()); Argv = {"clang", ADirExp.c_str()}; Res = cl::ExpandResponseFiles(Saver, cl::TokenizeGNUCommandLine, Argv); @@ -1125,7 +1124,6 @@ TEST(CommandLineTest, BadResponseFile) { ASSERT_EQ(2U, Argv.size()); ASSERT_STREQ(Argv[0], "clang"); ASSERT_STREQ(Argv[1], ADirExp.c_str()); -#endif } TEST(CommandLineTest, SetDefaultValue) { diff --git a/llvm/unittests/Support/Path.cpp b/llvm/unittests/Support/Path.cpp index 837ca03216f8..bdae7a8ee4b5 100644 --- a/llvm/unittests/Support/Path.cpp +++ b/llvm/unittests/Support/Path.cpp @@ -1757,6 +1757,28 @@ TEST_F(FileSystemTest, OpenFileForRead) { #endif } +TEST_F(FileSystemTest, OpenDirectoryAsFileForRead) { + std::string Buf(5, '?'); + Expected FD = fs::openNativeFileForRead(TestDirectory); +#ifdef _WIN32 + EXPECT_EQ(errorToErrorCode(FD.takeError()), errc::is_a_directory); +#else + ASSERT_THAT_EXPECTED(FD, Succeeded()); + auto Close = make_scope_exit([&] { fs::closeFile(*FD); }); + Expected BytesRead = + fs::readNativeFile(*FD, MutableArrayRef(&*Buf.begin(), Buf.size())); + EXPECT_EQ(errorToErrorCode(BytesRead.takeError()), errc::is_a_directory); +#endif +} + +TEST_F(FileSystemTest, OpenDirectoryAsFileForWrite) { + int FD; + std::error_code EC = fs::openFileForWrite(Twine(TestDirectory), FD); + if (!EC) + ::close(FD); + EXPECT_EQ(EC, errc::is_a_directory); +} + static void createFileWithData(const Twine &Path, bool ShouldExistBefore, fs::CreationDisposition Disp, StringRef Data) { int FD; -- GitLab From 26101e8c50e07ea53869b0c7f343d76bfed61b02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?= =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?= =?UTF-8?q?=E3=83=B3=29?= Date: Wed, 17 Apr 2024 15:42:09 -0700 Subject: [PATCH 109/862] [flang][cuda] Avoid crash by exiting the check if assignment is not usable (#89149) In the presence of other semantic error, `GetAssignment` would return a nullptr and therefore would make the rest of the check crash when trying to collect symbols. Exiting early when we have a nullptr so the compiler doesn't crash and user can get the meaningful semantic error. --- flang/lib/Semantics/check-cuda.cpp | 4 ++++ flang/test/Semantics/cuf11.cuf | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/flang/lib/Semantics/check-cuda.cpp b/flang/lib/Semantics/check-cuda.cpp index 2cb15437a235..fb1ebadd3785 100644 --- a/flang/lib/Semantics/check-cuda.cpp +++ b/flang/lib/Semantics/check-cuda.cpp @@ -488,6 +488,10 @@ void CUDAChecker::Enter(const parser::AssignmentStmt &x) { } const evaluate::Assignment *assign{semantics::GetAssignment(x)}; + if (!assign) { + return; + } + int nbLhs{evaluate::GetNbOfCUDASymbols(assign->lhs)}; int nbRhs{evaluate::GetNbOfCUDASymbols(assign->rhs)}; diff --git a/flang/test/Semantics/cuf11.cuf b/flang/test/Semantics/cuf11.cuf index b915c7246ed9..de7ff2974324 100644 --- a/flang/test/Semantics/cuf11.cuf +++ b/flang/test/Semantics/cuf11.cuf @@ -22,3 +22,11 @@ subroutine sub1() ahost = adev + adev end subroutine + +logical function compare_h(a,b) +!ERROR: Derived type 'h' not found + type(h) :: a, b +!ERROR: 'a' is not an object of derived type; it is implicitly typed +!ERROR: 'b' is not an object of derived type; it is implicitly typed + compare_h = (a%h .eq. b%h) +end -- GitLab From 823eb1a3252dd773f9c4d92093591f1b39ac27d4 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Wed, 17 Apr 2024 16:52:02 -0700 Subject: [PATCH 110/862] [SelectionDAG] Add some validation of (S/U)(ADD/SUB)O_CARRY nodes. (#89133) --- llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index ca0a95750ba8..f3441c9df0f8 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -9915,6 +9915,18 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, } break; } + case ISD::SADDO_CARRY: + case ISD::UADDO_CARRY: + case ISD::SSUBO_CARRY: + case ISD::USUBO_CARRY: + assert(VTList.NumVTs == 2 && Ops.size() == 3 && + "Invalid add/sub overflow op!"); + assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() && + Ops[0].getValueType() == Ops[1].getValueType() && + Ops[0].getValueType() == VTList.VTs[0] && + Ops[2].getValueType().isInteger() && + "Binary operator types must match!"); + break; case ISD::SMUL_LOHI: case ISD::UMUL_LOHI: { assert(VTList.NumVTs == 2 && Ops.size() == 2 && "Invalid mul lo/hi op!"); -- GitLab From 156ab4d4fb06be93b0cfce675e4cf86d330d879c Mon Sep 17 00:00:00 2001 From: Qizhi Hu <836744285@qq.com> Date: Thu, 18 Apr 2024 08:52:25 +0800 Subject: [PATCH 111/862] [Clang][Sema] set declaration invalid earlier to prevent crash in calculating record layout (#87173) Try to fix https://github.com/llvm/llvm-project/issues/75221 This crash caused by calculating record layout which contains a field declaration with dependent type. Make it invalid before it is a complete definition to prevent this crash. Define a new scope type to record this type alias and set the record declaration invalid when it is defined in a type alias template. Co-authored-by: huqizhi <836744285@qq.com> --- clang/docs/ReleaseNotes.rst | 2 ++ clang/include/clang/Sema/Scope.h | 6 ++++++ clang/lib/Parse/ParseDeclCXX.cpp | 5 +++++ clang/lib/Sema/SemaDecl.cpp | 7 +++++++ clang/test/SemaCXX/PR75221.cpp | 6 ++++++ 5 files changed, 26 insertions(+) create mode 100644 clang/test/SemaCXX/PR75221.cpp diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index c19ad9fba58f..6c51c2d1f483 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -536,6 +536,8 @@ Bug Fixes to C++ Support Fixes (#GH70604), (#GH79754), (#GH84163), (#GH84425), (#GH86054), (#GH86398), and (#GH86399). - Fix a crash when deducing ``auto`` from an invalid dereference (#GH88329). - Fix a crash in requires expression with templated base class member function. Fixes (#GH84020). +- Fix a crash caused by defined struct in a type alias template when the structure + has fields with dependent type. Fixes (#GH75221). Bug Fixes to AST Handling ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/clang/include/clang/Sema/Scope.h b/clang/include/clang/Sema/Scope.h index 099c2739e860..1752a25111a7 100644 --- a/clang/include/clang/Sema/Scope.h +++ b/clang/include/clang/Sema/Scope.h @@ -156,6 +156,9 @@ public: /// This is the scope of an OpenACC Compute Construct, which restricts /// jumping into/out of it. OpenACCComputeConstructScope = 0x10000000, + + /// This is a scope of type alias declaration. + TypeAliasScope = 0x20000000, }; private: @@ -580,6 +583,9 @@ public: /// if/switch/while/for statement. bool isControlScope() const { return getFlags() & Scope::ControlScope; } + /// Determine whether this scope is a type alias scope. + bool isTypeAliasScope() const { return getFlags() & Scope::TypeAliasScope; } + /// Returns if rhs has a higher scope depth than this. /// /// The caller is responsible for calling this only if one of the two scopes diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp index 51fd64b2d01a..8e0e86824829 100644 --- a/clang/lib/Parse/ParseDeclCXX.cpp +++ b/clang/lib/Parse/ParseDeclCXX.cpp @@ -799,6 +799,11 @@ Parser::DeclGroupPtrTy Parser::ParseUsingDeclaration( ProhibitAttributes(PrefixAttrs); Decl *DeclFromDeclSpec = nullptr; + Scope *CurScope = getCurScope(); + if (CurScope) + CurScope->setFlags(Scope::ScopeFlags::TypeAliasScope | + CurScope->getFlags()); + Decl *AD = ParseAliasDeclarationAfterDeclarator( TemplateInfo, UsingLoc, D, DeclEnd, AS, Attrs, &DeclFromDeclSpec); return Actions.ConvertDeclToDeclGroup(AD, DeclFromDeclSpec); diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp index 1bde99d6fce7..356abe09a5ca 100644 --- a/clang/lib/Sema/SemaDecl.cpp +++ b/clang/lib/Sema/SemaDecl.cpp @@ -19529,6 +19529,13 @@ void Sema::ActOnFields(Scope *S, SourceLocation RecLoc, Decl *EnclosingDecl, // Okay, we successfully defined 'Record'. if (Record) { bool Completed = false; + if (S) { + Scope *Parent = S->getParent(); + if (Parent && Parent->isTypeAliasScope() && + Parent->isTemplateParamScope()) + Record->setInvalidDecl(); + } + if (CXXRecord) { if (!CXXRecord->isInvalidDecl()) { // Set access bits correctly on the directly-declared conversions. diff --git a/clang/test/SemaCXX/PR75221.cpp b/clang/test/SemaCXX/PR75221.cpp new file mode 100644 index 000000000000..b342e347c560 --- /dev/null +++ b/clang/test/SemaCXX/PR75221.cpp @@ -0,0 +1,6 @@ +// RUN: %clang_cc1 -verify -std=c++11 -fsyntax-only %s + +template using foo = struct foo { // expected-error {{'foo' cannot be defined in a type alias template}} + T size = 0; +}; +foo a; -- GitLab From 888836930b1ed30b2095d45fcffc482560cb0e39 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Thu, 18 Apr 2024 10:15:25 +0900 Subject: [PATCH 112/862] Revert "[SLP]Attempt to vectorize long stores, if short one failed." This reverts commit 6f7160eedb2db02f37d4ffd52fff7b0cf88b3fdc. This still causes large compile-time regressions in some cases. --- .../Transforms/Vectorize/SLPVectorizer.cpp | 106 ++++++------------ .../Transforms/SLPVectorizer/X86/pr46983.ll | 46 ++++++-- 2 files changed, 72 insertions(+), 80 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 806e8085038b..7694627c3b04 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -15164,6 +15164,10 @@ bool SLPVectorizerPass::vectorizeStores(ArrayRef Stores, BoUpSLP::ValueSet VectorizedStores; bool Changed = false; + // Stores the pair of stores (first_store, last_store) in a range, that were + // already tried to be vectorized. Allows to skip the store ranges that were + // already tried to be vectorized but the attempts were unsuccessful. + DenseSet> TriedSequences; struct StoreDistCompare { bool operator()(const std::pair &Op1, const std::pair &Op2) const { @@ -15205,10 +15209,8 @@ bool SLPVectorizerPass::vectorizeStores(ArrayRef Stores, Type *ValueTy = StoreTy; if (auto *Trunc = dyn_cast(Store->getValueOperand())) ValueTy = Trunc->getSrcTy(); - unsigned MinVF = std::max( - 2, PowerOf2Ceil(TTI->getStoreMinimumVF( - R.getMinVF(DL->getTypeStoreSizeInBits(StoreTy)), StoreTy, - ValueTy))); + unsigned MinVF = PowerOf2Ceil(TTI->getStoreMinimumVF( + R.getMinVF(DL->getTypeStoreSizeInBits(StoreTy)), StoreTy, ValueTy)); if (MaxVF < MinVF) { LLVM_DEBUG(dbgs() << "SLP: Vectorization infeasible as MaxVF (" << MaxVF @@ -15234,74 +15236,40 @@ bool SLPVectorizerPass::vectorizeStores(ArrayRef Stores, VF = Size > MaxVF ? NonPowerOf2VF : Size; Size *= 2; }); - unsigned End = Operands.size(); - unsigned Repeat = 0; - constexpr unsigned MaxAttempts = 2; - SmallBitVector Range(Operands.size()); - while (true) { - ++Repeat; - for (unsigned Size : CandidateVFs) { - int StartIdx = Range.find_first_unset(); - while (StartIdx != -1) { - int EndIdx = Range.find_next(StartIdx); - unsigned Sz = EndIdx == -1 ? End : EndIdx; - for (unsigned Cnt = StartIdx; Cnt + Size <= Sz;) { - ArrayRef Slice = ArrayRef(Operands).slice(Cnt, Size); - assert(all_of(Slice, - [&](Value *V) { - return cast(V) - ->getValueOperand() - ->getType() == - cast(Slice.front()) - ->getValueOperand() - ->getType(); - }) && - "Expected all operands of same type."); - if (vectorizeStoreChain(Slice, R, Cnt, MinVF)) { - // Mark the vectorized stores so that we don't vectorize them - // again. - VectorizedStores.insert(Slice.begin(), Slice.end()); - // Mark the vectorized stores so that we don't vectorize them - // again. - Changed = true; - // If we vectorized initial block, no need to try to vectorize - // it again. - Range.set(Cnt, Cnt + Size); - if (Cnt < StartIdx + MinVF) - Range.set(StartIdx, Cnt); - if (Cnt > EndIdx - Size - MinVF) { - Range.set(Cnt + Size, EndIdx); - End = Cnt; - } - Cnt += Size; - continue; - } - ++Cnt; - } - if (Sz >= End) - break; - StartIdx = Range.find_next_unset(EndIdx); + unsigned StartIdx = 0; + for (unsigned Size : CandidateVFs) { + for (unsigned Cnt = StartIdx, E = Operands.size(); Cnt + Size <= E;) { + ArrayRef Slice = ArrayRef(Operands).slice(Cnt, Size); + assert( + all_of( + Slice, + [&](Value *V) { + return cast(V)->getValueOperand()->getType() == + cast(Slice.front()) + ->getValueOperand() + ->getType(); + }) && + "Expected all operands of same type."); + if (!VectorizedStores.count(Slice.front()) && + !VectorizedStores.count(Slice.back()) && + TriedSequences.insert(std::make_pair(Slice.front(), Slice.back())) + .second && + vectorizeStoreChain(Slice, R, Cnt, MinVF)) { + // Mark the vectorized stores so that we don't vectorize them again. + VectorizedStores.insert(Slice.begin(), Slice.end()); + Changed = true; + // If we vectorized initial block, no need to try to vectorize it + // again. + if (Cnt == StartIdx) + StartIdx += Size; + Cnt += Size; + continue; } + ++Cnt; } - // All values vectorize - exit. - if (Range.all()) - break; - // Check if tried all attempts or no need for the last attempts at all. - if (Repeat >= MaxAttempts) - break; - constexpr unsigned MaxVFScale = 4; - constexpr unsigned StoresLimit = 16; - const unsigned MaxTotalNum = std::min( - std::max(StoresLimit, MaxVFScale * MaxVF), - bit_floor(static_cast(Range.find_last_unset() - - Range.find_first_unset() + 1))); - if (MaxVF >= MaxTotalNum) + // Check if the whole array was vectorized already - exit. + if (StartIdx >= Operands.size()) break; - // Last attempt to vectorize max number of elements, if all previous - // attempts were unsuccessful because of the cost issues. - CandidateVFs.clear(); - for (unsigned Size = MaxTotalNum; Size > MaxVF; Size /= 2) - CandidateVFs.push_back(Size); } } }; diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll index 3deab0975ce7..75505f632a43 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll @@ -100,17 +100,41 @@ define void @store_i8(ptr nocapture %0, i32 %1, i32 %2) { define void @store_i64(ptr nocapture %0, i32 %1, i32 %2) { ; SSE-LABEL: @store_i64( ; SSE-NEXT: [[TMP4:%.*]] = zext i32 [[TMP1:%.*]] to i64 -; SSE-NEXT: [[TMP5:%.*]] = load <4 x i64>, ptr [[TMP0:%.*]], align 8, !tbaa [[TBAA5:![0-9]+]] -; SSE-NEXT: [[TMP6:%.*]] = insertelement <4 x i64> poison, i64 [[TMP4]], i64 0 -; SSE-NEXT: [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP6]], <4 x i64> poison, <4 x i32> zeroinitializer -; SSE-NEXT: [[TMP8:%.*]] = mul <4 x i64> [[TMP5]], [[TMP7]] -; SSE-NEXT: [[TMP9:%.*]] = lshr <4 x i64> [[TMP8]], -; SSE-NEXT: [[TMP10:%.*]] = trunc <4 x i64> [[TMP9]] to <4 x i32> -; SSE-NEXT: [[TMP11:%.*]] = icmp ult <4 x i32> [[TMP10]], -; SSE-NEXT: [[TMP12:%.*]] = trunc <4 x i64> [[TMP9]] to <4 x i32> -; SSE-NEXT: [[TMP13:%.*]] = select <4 x i1> [[TMP11]], <4 x i32> [[TMP12]], <4 x i32> -; SSE-NEXT: [[TMP14:%.*]] = zext <4 x i32> [[TMP13]] to <4 x i64> -; SSE-NEXT: store <4 x i64> [[TMP14]], ptr [[TMP0]], align 8, !tbaa [[TBAA5]] +; SSE-NEXT: [[TMP5:%.*]] = load i64, ptr [[TMP0:%.*]], align 8, !tbaa [[TBAA5:![0-9]+]] +; SSE-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], [[TMP4]] +; SSE-NEXT: [[TMP7:%.*]] = lshr i64 [[TMP6]], 15 +; SSE-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP7]] to i32 +; SSE-NEXT: [[TMP9:%.*]] = icmp ult i32 [[TMP8]], 255 +; SSE-NEXT: [[TMP10:%.*]] = and i64 [[TMP7]], 4294967295 +; SSE-NEXT: [[TMP11:%.*]] = select i1 [[TMP9]], i64 [[TMP10]], i64 255 +; SSE-NEXT: store i64 [[TMP11]], ptr [[TMP0]], align 8, !tbaa [[TBAA5]] +; SSE-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 8 +; SSE-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP12]], align 8, !tbaa [[TBAA5]] +; SSE-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], [[TMP4]] +; SSE-NEXT: [[TMP15:%.*]] = lshr i64 [[TMP14]], 15 +; SSE-NEXT: [[TMP16:%.*]] = trunc i64 [[TMP15]] to i32 +; SSE-NEXT: [[TMP17:%.*]] = icmp ult i32 [[TMP16]], 255 +; SSE-NEXT: [[TMP18:%.*]] = and i64 [[TMP15]], 4294967295 +; SSE-NEXT: [[TMP19:%.*]] = select i1 [[TMP17]], i64 [[TMP18]], i64 255 +; SSE-NEXT: store i64 [[TMP19]], ptr [[TMP12]], align 8, !tbaa [[TBAA5]] +; SSE-NEXT: [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 16 +; SSE-NEXT: [[TMP21:%.*]] = load i64, ptr [[TMP20]], align 8, !tbaa [[TBAA5]] +; SSE-NEXT: [[TMP22:%.*]] = mul i64 [[TMP21]], [[TMP4]] +; SSE-NEXT: [[TMP23:%.*]] = lshr i64 [[TMP22]], 15 +; SSE-NEXT: [[TMP24:%.*]] = trunc i64 [[TMP23]] to i32 +; SSE-NEXT: [[TMP25:%.*]] = icmp ult i32 [[TMP24]], 255 +; SSE-NEXT: [[TMP26:%.*]] = and i64 [[TMP23]], 4294967295 +; SSE-NEXT: [[TMP27:%.*]] = select i1 [[TMP25]], i64 [[TMP26]], i64 255 +; SSE-NEXT: store i64 [[TMP27]], ptr [[TMP20]], align 8, !tbaa [[TBAA5]] +; SSE-NEXT: [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 24 +; SSE-NEXT: [[TMP29:%.*]] = load i64, ptr [[TMP28]], align 8, !tbaa [[TBAA5]] +; SSE-NEXT: [[TMP30:%.*]] = mul i64 [[TMP29]], [[TMP4]] +; SSE-NEXT: [[TMP31:%.*]] = lshr i64 [[TMP30]], 15 +; SSE-NEXT: [[TMP32:%.*]] = trunc i64 [[TMP31]] to i32 +; SSE-NEXT: [[TMP33:%.*]] = icmp ult i32 [[TMP32]], 255 +; SSE-NEXT: [[TMP34:%.*]] = and i64 [[TMP31]], 4294967295 +; SSE-NEXT: [[TMP35:%.*]] = select i1 [[TMP33]], i64 [[TMP34]], i64 255 +; SSE-NEXT: store i64 [[TMP35]], ptr [[TMP28]], align 8, !tbaa [[TBAA5]] ; SSE-NEXT: ret void ; ; AVX-LABEL: @store_i64( -- GitLab From 748ef7eccc8b997ee5553cc5aadf04d7c7d0556f Mon Sep 17 00:00:00 2001 From: "Yaxun (Sam) Liu" Date: Wed, 17 Apr 2024 21:44:12 -0400 Subject: [PATCH 113/862] [CUDA][HIP] Fix record layout on Windows (#87651) On windows, record layout should be consistent with host side, otherwise host code is not able to access fields of the record correctly. Fixes: https://github.com/llvm/llvm-project/issues/51031 Fixes: SWDEV-446010 --- clang/lib/AST/RecordLayoutBuilder.cpp | 5 + clang/test/CodeGenCUDA/record-layout.cu | 200 ++++++++++++++++++++++++ 2 files changed, 205 insertions(+) create mode 100644 clang/test/CodeGenCUDA/record-layout.cu diff --git a/clang/lib/AST/RecordLayoutBuilder.cpp b/clang/lib/AST/RecordLayoutBuilder.cpp index a3b7431f7ffd..d9bf62c2bbb0 100644 --- a/clang/lib/AST/RecordLayoutBuilder.cpp +++ b/clang/lib/AST/RecordLayoutBuilder.cpp @@ -2458,6 +2458,11 @@ static bool mustSkipTailPadding(TargetCXXABI ABI, const CXXRecordDecl *RD) { } static bool isMsLayout(const ASTContext &Context) { + // Check if it's CUDA device compilation; ensure layout consistency with host. + if (Context.getLangOpts().CUDA && Context.getLangOpts().CUDAIsDevice && + Context.getAuxTargetInfo()) + return Context.getAuxTargetInfo()->getCXXABI().isMicrosoft(); + return Context.getTargetInfo().getCXXABI().isMicrosoft(); } diff --git a/clang/test/CodeGenCUDA/record-layout.cu b/clang/test/CodeGenCUDA/record-layout.cu new file mode 100644 index 000000000000..dd34121ccb9d --- /dev/null +++ b/clang/test/CodeGenCUDA/record-layout.cu @@ -0,0 +1,200 @@ +// RUN: %clang_cc1 -triple x86_64-pc-windows-msvc -fdump-record-layouts \ +// RUN: -emit-llvm -o %t -xhip %s 2>&1 | FileCheck %s --check-prefix=AST +// RUN: cat %t | FileCheck --check-prefixes=CHECK,HOST %s +// RUN: %clang_cc1 -fcuda-is-device -triple amdgcn-amd-amdhsa -target-cpu gfx1100 \ +// RUN: -emit-llvm -fdump-record-layouts -aux-triple x86_64-pc-windows-msvc \ +// RUN: -o %t -xhip %s | FileCheck %s --check-prefix=AST +// RUN: cat %t | FileCheck --check-prefixes=CHECK,DEV %s + +#include "Inputs/cuda.h" + +// AST: *** Dumping AST Record Layout +// AST-LABEL: 0 | struct C +// AST-NEXT: 0 | struct A (base) (empty) +// AST-NEXT: 1 | struct B (base) (empty) +// AST-NEXT: 4 | int i +// AST-NEXT: | [sizeof=8, align=4, +// AST-NEXT: | nvsize=8, nvalign=4] + +// CHECK: %struct.C = type { [4 x i8], i32 } + +struct A {}; +struct B {}; +struct C : A, B { + int i; +}; + +// AST: *** Dumping AST Record Layout +// AST-LABEL: 0 | struct I +// AST-NEXT: 0 | (I vftable pointer) +// AST-NEXT: 8 | int i +// AST-NEXT: | [sizeof=16, align=8, +// AST-NEXT: | nvsize=16, nvalign=8] + +// AST: *** Dumping AST Record Layout +// AST-LABEL: 0 | struct J +// AST-NEXT: 0 | struct I (primary base) +// AST-NEXT: 0 | (I vftable pointer) +// AST-NEXT: 8 | int i +// AST-NEXT: 16 | int j +// AST-NEXT: | [sizeof=24, align=8, +// AST-NEXT: | nvsize=24, nvalign=8] + +// CHECK: %struct.I = type { ptr, i32 } +// CHECK: %struct.J = type { %struct.I, i32 } + +// HOST: @0 = private unnamed_addr constant { [4 x ptr] } { [4 x ptr] [ptr @"??_R4J@@6B@", ptr @"?f@J@@UEAAXXZ", ptr null, ptr @"?h@J@@UEAAXXZ"] }, comdat($"??_7J@@6B@") +// HOST: @1 = private unnamed_addr constant { [4 x ptr] } { [4 x ptr] [ptr @"??_R4I@@6B@", ptr @_purecall, ptr null, ptr @_purecall] }, comdat($"??_7I@@6B@") +// HOST: @"??_7J@@6B@" = unnamed_addr alias ptr, getelementptr inbounds ({ [4 x ptr] }, ptr @0, i32 0, i32 0, i32 1) +// HOST: @"??_7I@@6B@" = unnamed_addr alias ptr, getelementptr inbounds ({ [4 x ptr] }, ptr @1, i32 0, i32 0, i32 1) + +// DEV: @_ZTV1J = linkonce_odr unnamed_addr addrspace(1) constant { [5 x ptr addrspace(1)] } { [5 x ptr addrspace(1)] [ptr addrspace(1) null, ptr addrspace(1) null, ptr addrspace(1) null, ptr addrspace(1) addrspacecast (ptr @_ZN1J1gEv to ptr addrspace(1)), ptr addrspace(1) addrspacecast (ptr @_ZN1J1hEv to ptr addrspace(1))] }, comdat, align 8 +// DEV: @_ZTV1I = linkonce_odr unnamed_addr addrspace(1) constant { [5 x ptr addrspace(1)] } { [5 x ptr addrspace(1)] [ptr addrspace(1) null, ptr addrspace(1) null, ptr addrspace(1) null, ptr addrspace(1) addrspacecast (ptr @__cxa_pure_virtual to ptr addrspace(1)), ptr addrspace(1) addrspacecast (ptr @__cxa_pure_virtual to ptr addrspace(1))] }, comdat, align 8 +struct I { + virtual void f() = 0; + __device__ virtual void g() = 0; + __device__ __host__ virtual void h() = 0; + int i; +}; + +struct J : I { + void f() override {} + __device__ void g() override {} + __device__ __host__ void h() override {} + int j; +}; + +// DEV: define dso_local amdgpu_kernel void @_Z8C_kernel1C(ptr addrspace(4) noundef byref(%struct.C) align 4 %0) +// DEV: %coerce = alloca %struct.C, align 4, addrspace(5) +// DEV: %c = addrspacecast ptr addrspace(5) %coerce to ptr +// DEV: call void @llvm.memcpy.p0.p4.i64(ptr align 4 %c, ptr addrspace(4) align 4 %0, i64 8, i1 false) +// DEV: %i = getelementptr inbounds %struct.C, ptr %c, i32 0, i32 1 +// DEV: store i32 1, ptr %i, align 4 + +__global__ void C_kernel(C c) +{ + c.i = 1; +} + +// HOST-LABEL: define dso_local void @"?test_C@@YAXXZ"() +// HOST: %c = alloca %struct.C, align 4 +// HOST: %i = getelementptr inbounds %struct.C, ptr %c, i32 0, i32 1 +// HOST: store i32 11, ptr %i, align 4 + +void test_C() { + C c; + c.i = 11; + C_kernel<<<1, 1>>>(c); +} + +// DEV: define dso_local void @_Z5J_devP1J(ptr noundef %j) +// DEV: %j.addr = alloca ptr, align 8, addrspace(5) +// DEV: %j.addr.ascast = addrspacecast ptr addrspace(5) %j.addr to ptr +// DEV: store ptr %j, ptr %j.addr.ascast, align 8 +// DEV: %0 = load ptr, ptr %j.addr.ascast, align 8 +// DEV: %i = getelementptr inbounds %struct.I, ptr %0, i32 0, i32 1 +// DEV: store i32 2, ptr %i, align 8 +// DEV: %1 = load ptr, ptr %j.addr.ascast, align 8 +// DEV: %j1 = getelementptr inbounds %struct.J, ptr %1, i32 0, i32 1 +// DEV: store i32 3, ptr %j1, align 8 +// DEV: %2 = load ptr, ptr %j.addr.ascast, align 8 +// DEV: %vtable = load ptr addrspace(1), ptr %2, align 8 +// DEV: %vfn = getelementptr inbounds ptr addrspace(1), ptr addrspace(1) %vtable, i64 1 +// DEV: %3 = load ptr addrspace(1), ptr addrspace(1) %vfn, align 8 +// DEV: call addrspace(1) void %3(ptr noundef nonnull align 8 dereferenceable(24) %2) +// DEV: %4 = load ptr, ptr %j.addr.ascast, align 8 +// DEV: %vtable2 = load ptr addrspace(1), ptr %4, align 8 +// DEV: %vfn3 = getelementptr inbounds ptr addrspace(1), ptr addrspace(1) %vtable2, i64 2 +// DEV: %5 = load ptr addrspace(1), ptr addrspace(1) %vfn3, align 8 +// DEV: call addrspace(1) void %5(ptr noundef nonnull align 8 dereferenceable(24) %4) + +__device__ void J_dev(J *j) { + j->i = 2; + j->j = 3; + j->g(); + j->h(); +} + +// DEV: define dso_local amdgpu_kernel void @_Z8J_kernelv() +// DEV: %j = alloca %struct.J, align 8, addrspace(5) +// DEV: %j.ascast = addrspacecast ptr addrspace(5) %j to ptr +// DEV: call void @_ZN1JC1Ev(ptr noundef nonnull align 8 dereferenceable(24) %j.ascast) +// DEV: call void @_Z5J_devP1J(ptr noundef %j.ascast) + +__global__ void J_kernel() { + J j; + J_dev(&j); +} + +// HOST-LABEL: define dso_local void @"?J_host@@YAXPEAUJ@@@Z"(ptr noundef %j) +// HOST: %0 = load ptr, ptr %j.addr, align 8 +// HOST: %i = getelementptr inbounds %struct.I, ptr %0, i32 0, i32 1 +// HOST: store i32 12, ptr %i, align 8 +// HOST: %1 = load ptr, ptr %j.addr, align 8 +// HOST: %j1 = getelementptr inbounds %struct.J, ptr %1, i32 0, i32 1 +// HOST: store i32 13, ptr %j1, align 8 +// HOST: %2 = load ptr, ptr %j.addr, align 8 +// HOST: %vtable = load ptr, ptr %2, align 8 +// HOST: %vfn = getelementptr inbounds ptr, ptr %vtable, i64 0 +// HOST: %3 = load ptr, ptr %vfn, align 8 +// HOST: call void %3(ptr noundef nonnull align 8 dereferenceable(24) %2) +// HOST: %4 = load ptr, ptr %j.addr, align 8 +// HOST: %vtable2 = load ptr, ptr %4, align 8 +// HOST: %vfn3 = getelementptr inbounds ptr, ptr %vtable2, i64 2 +// HOST: %5 = load ptr, ptr %vfn3, align 8 +// HOST: call void %5(ptr noundef nonnull align 8 dereferenceable(24) %4) + +void J_host(J *j) { + j->i = 12; + j->j = 13; + j->f(); + j->h(); +} + +// HOST: define dso_local void @"?test_J@@YAXXZ"() +// HOST: %j = alloca %struct.J, align 8 +// HOST: %call = call noundef ptr @"??0J@@QEAA@XZ"(ptr noundef nonnull align 8 dereferenceable(24) %j) +// HOST: call void @"?J_host@@YAXPEAUJ@@@Z"(ptr noundef %j) + +void test_J() { + J j; + J_host(&j); + J_kernel<<<1, 1>>>(); +} + +// HOST: define linkonce_odr dso_local noundef ptr @"??0J@@QEAA@XZ"(ptr noundef nonnull returned align 8 dereferenceable(24) %this) +// HOST: %this.addr = alloca ptr, align 8 +// HOST: store ptr %this, ptr %this.addr, align 8 +// HOST: %this1 = load ptr, ptr %this.addr, align 8 +// HOST: %call = call noundef ptr @"??0I@@QEAA@XZ"(ptr noundef nonnull align 8 dereferenceable(16) %this1) #5 +// HOST: store ptr @"??_7J@@6B@", ptr %this1, align 8 +// HOST: ret ptr %this1 + +// HOST: define linkonce_odr dso_local noundef ptr @"??0I@@QEAA@XZ"(ptr noundef nonnull returned align 8 dereferenceable(16) %this) +// HOST: %this.addr = alloca ptr, align 8 +// HOST: store ptr %this, ptr %this.addr, align 8 +// HOST: %this1 = load ptr, ptr %this.addr, align 8 +// HOST: store ptr @"??_7I@@6B@", ptr %this1, align 8 +// HOST: ret ptr %this1 + +// DEV: define linkonce_odr void @_ZN1JC1Ev(ptr noundef nonnull align 8 dereferenceable(24) %this) +// DEV: %this.addr = alloca ptr, align 8, addrspace(5) +// DEV: %this.addr.ascast = addrspacecast ptr addrspace(5) %this.addr to ptr +// DEV: store ptr %this, ptr %this.addr.ascast, align 8 +// DEV: %this1 = load ptr, ptr %this.addr.ascast, align 8 +// DEV: call void @_ZN1JC2Ev(ptr noundef nonnull align 8 dereferenceable(24) %this1) + +// DEV: define linkonce_odr void @_ZN1JC2Ev(ptr noundef nonnull align 8 dereferenceable(24) %this) +// DEV: %this.addr = alloca ptr, align 8, addrspace(5) +// DEV: %this.addr.ascast = addrspacecast ptr addrspace(5) %this.addr to ptr +// DEV: store ptr %this, ptr %this.addr.ascast, align 8 +// DEV: %this1 = load ptr, ptr %this.addr.ascast, align 8 +// DEV: call void @_ZN1IC2Ev(ptr noundef nonnull align 8 dereferenceable(16) %this1) +// DEV: store ptr addrspace(1) getelementptr inbounds inrange(-16, 24) ({ [5 x ptr addrspace(1)] }, ptr addrspace(1) @_ZTV1J, i32 0, i32 0, i32 2), ptr %this1, align 8 + +// DEV: define linkonce_odr void @_ZN1IC2Ev(ptr noundef nonnull align 8 dereferenceable(16) %this) +// DEV: %this.addr = alloca ptr, align 8, addrspace(5) +// DEV: %this.addr.ascast = addrspacecast ptr addrspace(5) %this.addr to ptr +// DEV: store ptr %this, ptr %this.addr.ascast, align 8 +// DEV: %this1 = load ptr, ptr %this.addr.ascast, align 8 +// DEV: store ptr addrspace(1) getelementptr inbounds inrange(-16, 24) ({ [5 x ptr addrspace(1)] }, ptr addrspace(1) @_ZTV1I, i32 0, i32 0, i32 2), ptr %this1, align 8 -- GitLab From 097b68ff067565af744b5b0327732d7c199710ab Mon Sep 17 00:00:00 2001 From: Shih-Po Hung Date: Thu, 18 Apr 2024 09:44:31 +0800 Subject: [PATCH 114/862] [RISCV][TTI] Refine the cost of FCmp (#88833) This patch introduces following changes - Support all fp predicates - Use the Val type to estimate the latency/throughput cost - Assign a cost of 1 for mask operations as LMULCost for mask types cannot be correctly estimated. --- .../Target/RISCV/RISCVTargetTransformInfo.cpp | 48 +- llvm/test/Analysis/CostModel/RISCV/rvv-cmp.ll | 582 +++++++++++------- .../Analysis/CostModel/RISCV/rvv-select.ll | 8 +- 3 files changed, 396 insertions(+), 242 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp index 56f5bd8794ae..ce26e61880fd 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp @@ -91,6 +91,10 @@ RISCVTTIImpl::getRISCVInstructionCost(ArrayRef OpCodes, MVT VT, case RISCV::VMV_S_X: case RISCV::VFMV_F_S: case RISCV::VFMV_S_F: + case RISCV::VMOR_MM: + case RISCV::VMXOR_MM: + case RISCV::VMAND_MM: + case RISCV::VMANDN_MM: case RISCV::VMNAND_MM: case RISCV::VCPOP_M: Cost += 1; @@ -1383,7 +1387,13 @@ InstructionCost RISCVTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy, getRISCVInstructionCost(RISCV::VMSLT_VV, LT.second, CostKind); } - if ((Opcode == Instruction::FCmp) && ValTy->isVectorTy()) { + if ((Opcode == Instruction::FCmp) && ValTy->isVectorTy() && + CmpInst::isFPPredicate(VecPred)) { + + // Use VMXOR_MM and VMXNOR_MM to generate all true/false mask + if ((VecPred == CmpInst::FCMP_FALSE) || (VecPred == CmpInst::FCMP_TRUE)) + return getRISCVInstructionCost(RISCV::VMXOR_MM, LT.second, CostKind); + // If we do not support the input floating point vector type, use the base // one which will calculate as: // ScalarizeCost + Num * Cost for fixed vector, @@ -1393,16 +1403,34 @@ InstructionCost RISCVTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy, (ValTy->getScalarSizeInBits() == 64 && !ST->hasVInstructionsF64())) return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind, I); + + // Assuming vector fp compare and mask instructions are all the same cost + // until a need arises to differentiate them. switch (VecPred) { - // Support natively. - case CmpInst::FCMP_OEQ: - case CmpInst::FCMP_OGT: - case CmpInst::FCMP_OGE: - case CmpInst::FCMP_OLT: - case CmpInst::FCMP_OLE: - case CmpInst::FCMP_UNE: - return LT.first * 1; - // TODO: Other comparisons? + case CmpInst::FCMP_ONE: // vmflt.vv + vmflt.vv + vmor.mm + case CmpInst::FCMP_ORD: // vmfeq.vv + vmfeq.vv + vmand.mm + case CmpInst::FCMP_UNO: // vmfne.vv + vmfne.vv + vmor.mm + case CmpInst::FCMP_UEQ: // vmflt.vv + vmflt.vv + vmnor.mm + return LT.first * getRISCVInstructionCost( + {RISCV::VMFLT_VV, RISCV::VMFLT_VV, RISCV::VMOR_MM}, + LT.second, CostKind); + + case CmpInst::FCMP_UGT: // vmfle.vv + vmnot.m + case CmpInst::FCMP_UGE: // vmflt.vv + vmnot.m + case CmpInst::FCMP_ULT: // vmfle.vv + vmnot.m + case CmpInst::FCMP_ULE: // vmflt.vv + vmnot.m + return LT.first * + getRISCVInstructionCost({RISCV::VMFLT_VV, RISCV::VMNAND_MM}, + LT.second, CostKind); + + case CmpInst::FCMP_OEQ: // vmfeq.vv + case CmpInst::FCMP_OGT: // vmflt.vv + case CmpInst::FCMP_OGE: // vmfle.vv + case CmpInst::FCMP_OLT: // vmflt.vv + case CmpInst::FCMP_OLE: // vmfle.vv + case CmpInst::FCMP_UNE: // vmfne.vv + return LT.first * + getRISCVInstructionCost(RISCV::VMFLT_VV, LT.second, CostKind); default: break; } diff --git a/llvm/test/Analysis/CostModel/RISCV/rvv-cmp.ll b/llvm/test/Analysis/CostModel/RISCV/rvv-cmp.ll index caa6d6f483a2..8aa78ec95e8a 100644 --- a/llvm/test/Analysis/CostModel/RISCV/rvv-cmp.ll +++ b/llvm/test/Analysis/CostModel/RISCV/rvv-cmp.ll @@ -878,28 +878,28 @@ define void @fcmp_oeq() { ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fcmp oeq <2 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f16 = fcmp oeq <4 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f16 = fcmp oeq <8 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f16 = fcmp oeq <16 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v16f16 = fcmp oeq <16 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f16 = fcmp oeq undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f16 = fcmp oeq undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f16 = fcmp oeq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f16 = fcmp oeq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f16 = fcmp oeq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv8f16 = fcmp oeq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv16f16 = fcmp oeq undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f32 = fcmp oeq <2 x float> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f32 = fcmp oeq <4 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f32 = fcmp oeq <8 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f32 = fcmp oeq <16 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8f32 = fcmp oeq <8 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v16f32 = fcmp oeq <16 x float> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32 = fcmp oeq undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32 = fcmp oeq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32 = fcmp oeq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f32 = fcmp oeq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f32 = fcmp oeq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4f32 = fcmp oeq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv8f32 = fcmp oeq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv16f32 = fcmp oeq undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f64 = fcmp oeq <2 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f64 = fcmp oeq <4 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f64 = fcmp oeq <8 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4f64 = fcmp oeq <4 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8f64 = fcmp oeq <8 x double> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64 = fcmp oeq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f64 = fcmp oeq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f64 = fcmp oeq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f64 = fcmp oeq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2f64 = fcmp oeq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv4f64 = fcmp oeq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv8f64 = fcmp oeq undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %v2f16 = fcmp oeq <2 x half> undef, undef @@ -938,31 +938,31 @@ define void @fcmp_oeq() { define void @fcmp_one() { ; CHECK-LABEL: 'fcmp_one' -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fcmp one <2 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f16 = fcmp one <4 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f16 = fcmp one <8 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f16 = fcmp one <16 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f16 = fcmp one undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f16 = fcmp one undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f16 = fcmp one undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f16 = fcmp one undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f16 = fcmp one undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f32 = fcmp one <2 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f32 = fcmp one <4 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f32 = fcmp one <8 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f32 = fcmp one <16 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32 = fcmp one undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32 = fcmp one undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32 = fcmp one undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f32 = fcmp one undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f32 = fcmp one undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f64 = fcmp one <2 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f64 = fcmp one <4 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f64 = fcmp one <8 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64 = fcmp one undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f64 = fcmp one undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f64 = fcmp one undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f64 = fcmp one undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v2f16 = fcmp one <2 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4f16 = fcmp one <4 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v8f16 = fcmp one <8 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v16f16 = fcmp one <16 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv1f16 = fcmp one undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv2f16 = fcmp one undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv4f16 = fcmp one undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv8f16 = fcmp one undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv16f16 = fcmp one undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v2f32 = fcmp one <2 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4f32 = fcmp one <4 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v8f32 = fcmp one <8 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %v16f32 = fcmp one <16 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv1f32 = fcmp one undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv2f32 = fcmp one undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv4f32 = fcmp one undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv8f32 = fcmp one undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %nxv16f32 = fcmp one undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v2f64 = fcmp one <2 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v4f64 = fcmp one <4 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %v8f64 = fcmp one <8 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv1f64 = fcmp one undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv2f64 = fcmp one undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv4f64 = fcmp one undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %nxv8f64 = fcmp one undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %v2f16 = fcmp one <2 x half> undef, undef @@ -1004,28 +1004,28 @@ define void @fcmp_olt() { ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fcmp olt <2 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f16 = fcmp olt <4 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f16 = fcmp olt <8 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f16 = fcmp olt <16 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v16f16 = fcmp olt <16 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f16 = fcmp olt undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f16 = fcmp olt undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f16 = fcmp olt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f16 = fcmp olt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f16 = fcmp olt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv8f16 = fcmp olt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv16f16 = fcmp olt undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f32 = fcmp olt <2 x float> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f32 = fcmp olt <4 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f32 = fcmp olt <8 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f32 = fcmp olt <16 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8f32 = fcmp olt <8 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v16f32 = fcmp olt <16 x float> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32 = fcmp olt undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32 = fcmp olt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32 = fcmp olt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f32 = fcmp olt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f32 = fcmp olt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4f32 = fcmp olt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv8f32 = fcmp olt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv16f32 = fcmp olt undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f64 = fcmp olt <2 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f64 = fcmp olt <4 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f64 = fcmp olt <8 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4f64 = fcmp olt <4 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8f64 = fcmp olt <8 x double> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64 = fcmp olt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f64 = fcmp olt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f64 = fcmp olt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f64 = fcmp olt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2f64 = fcmp olt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv4f64 = fcmp olt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv8f64 = fcmp olt undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %v2f16 = fcmp olt <2 x half> undef, undef @@ -1067,28 +1067,28 @@ define void @fcmp_ole() { ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fcmp ole <2 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f16 = fcmp ole <4 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f16 = fcmp ole <8 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f16 = fcmp ole <16 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v16f16 = fcmp ole <16 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f16 = fcmp ole undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f16 = fcmp ole undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f16 = fcmp ole undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f16 = fcmp ole undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f16 = fcmp ole undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv8f16 = fcmp ole undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv16f16 = fcmp ole undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f32 = fcmp ole <2 x float> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f32 = fcmp ole <4 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f32 = fcmp ole <8 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f32 = fcmp ole <16 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8f32 = fcmp ole <8 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v16f32 = fcmp ole <16 x float> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32 = fcmp ole undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32 = fcmp ole undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32 = fcmp ole undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f32 = fcmp ole undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f32 = fcmp ole undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4f32 = fcmp ole undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv8f32 = fcmp ole undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv16f32 = fcmp ole undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f64 = fcmp ole <2 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f64 = fcmp ole <4 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f64 = fcmp ole <8 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4f64 = fcmp ole <4 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8f64 = fcmp ole <8 x double> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64 = fcmp ole undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f64 = fcmp ole undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f64 = fcmp ole undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f64 = fcmp ole undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2f64 = fcmp ole undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv4f64 = fcmp ole undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv8f64 = fcmp ole undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %v2f16 = fcmp ole <2 x half> undef, undef @@ -1130,28 +1130,28 @@ define void @fcmp_ogt() { ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fcmp ogt <2 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f16 = fcmp ogt <4 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f16 = fcmp ogt <8 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f16 = fcmp ogt <16 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v16f16 = fcmp ogt <16 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f16 = fcmp ogt undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f16 = fcmp ogt undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f16 = fcmp ogt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f16 = fcmp ogt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f16 = fcmp ogt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv8f16 = fcmp ogt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv16f16 = fcmp ogt undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f32 = fcmp ogt <2 x float> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f32 = fcmp ogt <4 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f32 = fcmp ogt <8 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f32 = fcmp ogt <16 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8f32 = fcmp ogt <8 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v16f32 = fcmp ogt <16 x float> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32 = fcmp ogt undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32 = fcmp ogt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32 = fcmp ogt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f32 = fcmp ogt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f32 = fcmp ogt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4f32 = fcmp ogt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv8f32 = fcmp ogt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv16f32 = fcmp ogt undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f64 = fcmp ogt <2 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f64 = fcmp ogt <4 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f64 = fcmp ogt <8 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4f64 = fcmp ogt <4 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8f64 = fcmp ogt <8 x double> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64 = fcmp ogt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f64 = fcmp ogt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f64 = fcmp ogt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f64 = fcmp ogt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2f64 = fcmp ogt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv4f64 = fcmp ogt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv8f64 = fcmp ogt undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %v2f16 = fcmp ogt <2 x half> undef, undef @@ -1193,28 +1193,28 @@ define void @fcmp_oge() { ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fcmp oge <2 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f16 = fcmp oge <4 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f16 = fcmp oge <8 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f16 = fcmp oge <16 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v16f16 = fcmp oge <16 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f16 = fcmp oge undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f16 = fcmp oge undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f16 = fcmp oge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f16 = fcmp oge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f16 = fcmp oge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv8f16 = fcmp oge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv16f16 = fcmp oge undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f32 = fcmp oge <2 x float> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f32 = fcmp oge <4 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f32 = fcmp oge <8 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f32 = fcmp oge <16 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8f32 = fcmp oge <8 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v16f32 = fcmp oge <16 x float> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32 = fcmp oge undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32 = fcmp oge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32 = fcmp oge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f32 = fcmp oge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f32 = fcmp oge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4f32 = fcmp oge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv8f32 = fcmp oge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv16f32 = fcmp oge undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f64 = fcmp oge <2 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f64 = fcmp oge <4 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f64 = fcmp oge <8 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4f64 = fcmp oge <4 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8f64 = fcmp oge <8 x double> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64 = fcmp oge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f64 = fcmp oge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f64 = fcmp oge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f64 = fcmp oge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2f64 = fcmp oge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv4f64 = fcmp oge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv8f64 = fcmp oge undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %v2f16 = fcmp oge <2 x half> undef, undef @@ -1253,31 +1253,31 @@ define void @fcmp_oge() { define void @fcmp_ueq() { ; CHECK-LABEL: 'fcmp_ueq' -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fcmp ueq <2 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f16 = fcmp ueq <4 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f16 = fcmp ueq <8 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f16 = fcmp ueq <16 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f16 = fcmp ueq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f16 = fcmp ueq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f16 = fcmp ueq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f16 = fcmp ueq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f16 = fcmp ueq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f32 = fcmp ueq <2 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f32 = fcmp ueq <4 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f32 = fcmp ueq <8 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f32 = fcmp ueq <16 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32 = fcmp ueq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32 = fcmp ueq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32 = fcmp ueq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f32 = fcmp ueq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f32 = fcmp ueq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f64 = fcmp ueq <2 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f64 = fcmp ueq <4 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f64 = fcmp ueq <8 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64 = fcmp ueq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f64 = fcmp ueq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f64 = fcmp ueq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f64 = fcmp ueq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v2f16 = fcmp ueq <2 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4f16 = fcmp ueq <4 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v8f16 = fcmp ueq <8 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v16f16 = fcmp ueq <16 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv1f16 = fcmp ueq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv2f16 = fcmp ueq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv4f16 = fcmp ueq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv8f16 = fcmp ueq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv16f16 = fcmp ueq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v2f32 = fcmp ueq <2 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4f32 = fcmp ueq <4 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v8f32 = fcmp ueq <8 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %v16f32 = fcmp ueq <16 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv1f32 = fcmp ueq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv2f32 = fcmp ueq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv4f32 = fcmp ueq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv8f32 = fcmp ueq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %nxv16f32 = fcmp ueq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v2f64 = fcmp ueq <2 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v4f64 = fcmp ueq <4 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %v8f64 = fcmp ueq <8 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv1f64 = fcmp ueq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv2f64 = fcmp ueq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv4f64 = fcmp ueq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %nxv8f64 = fcmp ueq undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %v2f16 = fcmp ueq <2 x half> undef, undef @@ -1319,28 +1319,28 @@ define void @fcmp_une() { ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fcmp une <2 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f16 = fcmp une <4 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f16 = fcmp une <8 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f16 = fcmp une <16 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v16f16 = fcmp une <16 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f16 = fcmp une undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f16 = fcmp une undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f16 = fcmp une undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f16 = fcmp une undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f16 = fcmp une undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv8f16 = fcmp une undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv16f16 = fcmp une undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f32 = fcmp une <2 x float> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f32 = fcmp une <4 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f32 = fcmp une <8 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f32 = fcmp une <16 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8f32 = fcmp une <8 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v16f32 = fcmp une <16 x float> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32 = fcmp une undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32 = fcmp une undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32 = fcmp une undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f32 = fcmp une undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f32 = fcmp une undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4f32 = fcmp une undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv8f32 = fcmp une undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv16f32 = fcmp une undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f64 = fcmp une <2 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f64 = fcmp une <4 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f64 = fcmp une <8 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4f64 = fcmp une <4 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8f64 = fcmp une <8 x double> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64 = fcmp une undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f64 = fcmp une undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f64 = fcmp une undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f64 = fcmp une undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2f64 = fcmp une undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv4f64 = fcmp une undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv8f64 = fcmp une undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %v2f16 = fcmp une <2 x half> undef, undef @@ -1379,31 +1379,31 @@ define void @fcmp_une() { define void @fcmp_ult() { ; CHECK-LABEL: 'fcmp_ult' -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fcmp ult <2 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f16 = fcmp ult <4 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f16 = fcmp ult <8 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f16 = fcmp ult <16 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f16 = fcmp ult undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f16 = fcmp ult undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f16 = fcmp ult undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f16 = fcmp ult undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f16 = fcmp ult undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f32 = fcmp ult <2 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f32 = fcmp ult <4 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f32 = fcmp ult <8 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f32 = fcmp ult <16 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32 = fcmp ult undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32 = fcmp ult undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32 = fcmp ult undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f32 = fcmp ult undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f32 = fcmp ult undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f64 = fcmp ult <2 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f64 = fcmp ult <4 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f64 = fcmp ult <8 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64 = fcmp ult undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f64 = fcmp ult undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f64 = fcmp ult undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f64 = fcmp ult undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f16 = fcmp ult <2 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4f16 = fcmp ult <4 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8f16 = fcmp ult <8 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v16f16 = fcmp ult <16 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv1f16 = fcmp ult undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2f16 = fcmp ult undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4f16 = fcmp ult undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv8f16 = fcmp ult undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv16f16 = fcmp ult undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f32 = fcmp ult <2 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4f32 = fcmp ult <4 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v8f32 = fcmp ult <8 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v16f32 = fcmp ult <16 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv1f32 = fcmp ult undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2f32 = fcmp ult undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv4f32 = fcmp ult undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv8f32 = fcmp ult undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv16f32 = fcmp ult undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f64 = fcmp ult <2 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4f64 = fcmp ult <4 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v8f64 = fcmp ult <8 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv1f64 = fcmp ult undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv2f64 = fcmp ult undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv4f64 = fcmp ult undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv8f64 = fcmp ult undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %v2f16 = fcmp ult <2 x half> undef, undef @@ -1442,31 +1442,31 @@ define void @fcmp_ult() { define void @fcmp_ule() { ; CHECK-LABEL: 'fcmp_ule' -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fcmp ule <2 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f16 = fcmp ule <4 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f16 = fcmp ule <8 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f16 = fcmp ule <16 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f16 = fcmp ule undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f16 = fcmp ule undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f16 = fcmp ule undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f16 = fcmp ule undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f16 = fcmp ule undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f32 = fcmp ule <2 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f32 = fcmp ule <4 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f32 = fcmp ule <8 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f32 = fcmp ule <16 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32 = fcmp ule undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32 = fcmp ule undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32 = fcmp ule undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f32 = fcmp ule undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f32 = fcmp ule undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f64 = fcmp ule <2 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f64 = fcmp ule <4 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f64 = fcmp ule <8 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64 = fcmp ule undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f64 = fcmp ule undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f64 = fcmp ule undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f64 = fcmp ule undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f16 = fcmp ule <2 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4f16 = fcmp ule <4 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8f16 = fcmp ule <8 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v16f16 = fcmp ule <16 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv1f16 = fcmp ule undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2f16 = fcmp ule undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4f16 = fcmp ule undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv8f16 = fcmp ule undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv16f16 = fcmp ule undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f32 = fcmp ule <2 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4f32 = fcmp ule <4 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v8f32 = fcmp ule <8 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v16f32 = fcmp ule <16 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv1f32 = fcmp ule undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2f32 = fcmp ule undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv4f32 = fcmp ule undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv8f32 = fcmp ule undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv16f32 = fcmp ule undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f64 = fcmp ule <2 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4f64 = fcmp ule <4 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v8f64 = fcmp ule <8 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv1f64 = fcmp ule undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv2f64 = fcmp ule undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv4f64 = fcmp ule undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv8f64 = fcmp ule undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %v2f16 = fcmp ule <2 x half> undef, undef @@ -1505,31 +1505,31 @@ define void @fcmp_ule() { define void @fcmp_ugt() { ; CHECK-LABEL: 'fcmp_ugt' -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fcmp ugt <2 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f16 = fcmp ugt <4 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f16 = fcmp ugt <8 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f16 = fcmp ugt <16 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f16 = fcmp ugt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f16 = fcmp ugt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f16 = fcmp ugt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f16 = fcmp ugt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f16 = fcmp ugt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f32 = fcmp ugt <2 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f32 = fcmp ugt <4 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f32 = fcmp ugt <8 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f32 = fcmp ugt <16 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32 = fcmp ugt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32 = fcmp ugt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32 = fcmp ugt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f32 = fcmp ugt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f32 = fcmp ugt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f64 = fcmp ugt <2 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f64 = fcmp ugt <4 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f64 = fcmp ugt <8 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64 = fcmp ugt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f64 = fcmp ugt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f64 = fcmp ugt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f64 = fcmp ugt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f16 = fcmp ugt <2 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4f16 = fcmp ugt <4 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8f16 = fcmp ugt <8 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v16f16 = fcmp ugt <16 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv1f16 = fcmp ugt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2f16 = fcmp ugt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4f16 = fcmp ugt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv8f16 = fcmp ugt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv16f16 = fcmp ugt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f32 = fcmp ugt <2 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4f32 = fcmp ugt <4 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v8f32 = fcmp ugt <8 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v16f32 = fcmp ugt <16 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv1f32 = fcmp ugt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2f32 = fcmp ugt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv4f32 = fcmp ugt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv8f32 = fcmp ugt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv16f32 = fcmp ugt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f64 = fcmp ugt <2 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4f64 = fcmp ugt <4 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v8f64 = fcmp ugt <8 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv1f64 = fcmp ugt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv2f64 = fcmp ugt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv4f64 = fcmp ugt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv8f64 = fcmp ugt undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %v2f16 = fcmp ugt <2 x half> undef, undef @@ -1568,31 +1568,31 @@ define void @fcmp_ugt() { define void @fcmp_uge() { ; CHECK-LABEL: 'fcmp_uge' -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fcmp uge <2 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f16 = fcmp uge <4 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f16 = fcmp uge <8 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f16 = fcmp uge <16 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f16 = fcmp uge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f16 = fcmp uge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f16 = fcmp uge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f16 = fcmp uge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f16 = fcmp uge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f32 = fcmp uge <2 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f32 = fcmp uge <4 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f32 = fcmp uge <8 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f32 = fcmp uge <16 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32 = fcmp uge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32 = fcmp uge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32 = fcmp uge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f32 = fcmp uge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f32 = fcmp uge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f64 = fcmp uge <2 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f64 = fcmp uge <4 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f64 = fcmp uge <8 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64 = fcmp uge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f64 = fcmp uge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f64 = fcmp uge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f64 = fcmp uge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f16 = fcmp uge <2 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4f16 = fcmp uge <4 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8f16 = fcmp uge <8 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v16f16 = fcmp uge <16 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv1f16 = fcmp uge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2f16 = fcmp uge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4f16 = fcmp uge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv8f16 = fcmp uge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv16f16 = fcmp uge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f32 = fcmp uge <2 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4f32 = fcmp uge <4 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v8f32 = fcmp uge <8 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v16f32 = fcmp uge <16 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv1f32 = fcmp uge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2f32 = fcmp uge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv4f32 = fcmp uge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv8f32 = fcmp uge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv16f32 = fcmp uge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f64 = fcmp uge <2 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4f64 = fcmp uge <4 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v8f64 = fcmp uge <8 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv1f64 = fcmp uge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv2f64 = fcmp uge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv4f64 = fcmp uge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv8f64 = fcmp uge undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %v2f16 = fcmp uge <2 x half> undef, undef @@ -1628,3 +1628,129 @@ define void @fcmp_uge() { ret void } + +define void @fcmp_true() { +; CHECK-LABEL: 'fcmp_true' +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fcmp true <2 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f16 = fcmp true <4 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f16 = fcmp true <8 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f16 = fcmp true <16 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f16 = fcmp true undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f16 = fcmp true undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f16 = fcmp true undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f16 = fcmp true undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f16 = fcmp true undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f32 = fcmp true <2 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f32 = fcmp true <4 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f32 = fcmp true <8 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f32 = fcmp true <16 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32 = fcmp true undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32 = fcmp true undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32 = fcmp true undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f32 = fcmp true undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f32 = fcmp true undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f64 = fcmp true <2 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f64 = fcmp true <4 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f64 = fcmp true <8 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64 = fcmp true undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f64 = fcmp true undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f64 = fcmp true undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f64 = fcmp true undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; + %v2f16 = fcmp true <2 x half> undef, undef + %v4f16 = fcmp true <4 x half> undef, undef + %v8f16 = fcmp true <8 x half> undef, undef + %v16f16 = fcmp true <16 x half> undef, undef + + %nxv1f16 = fcmp true undef, undef + %nxv2f16 = fcmp true undef, undef + %nxv4f16 = fcmp true undef, undef + %nxv8f16 = fcmp true undef, undef + %nxv16f16 = fcmp true undef, undef + + %v2f32 = fcmp true <2 x float> undef, undef + %v4f32 = fcmp true <4 x float> undef, undef + %v8f32 = fcmp true <8 x float> undef, undef + %v16f32 = fcmp true <16 x float> undef, undef + + %nxv1f32 = fcmp true undef, undef + %nxv2f32 = fcmp true undef, undef + %nxv4f32 = fcmp true undef, undef + %nxv8f32 = fcmp true undef, undef + %nxv16f32 = fcmp true undef, undef + + %v2f64 = fcmp true <2 x double> undef, undef + %v4f64 = fcmp true <4 x double> undef, undef + %v8f64 = fcmp true <8 x double> undef, undef + + %nxv1f64 = fcmp true undef, undef + %nxv2f64 = fcmp true undef, undef + %nxv4f64 = fcmp true undef, undef + %nxv8f64 = fcmp true undef, undef + + ret void +} + +define void @fcmp_false() { +; CHECK-LABEL: 'fcmp_false' +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fcmp false <2 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f16 = fcmp false <4 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f16 = fcmp false <8 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f16 = fcmp false <16 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f16 = fcmp false undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f16 = fcmp false undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f16 = fcmp false undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f16 = fcmp false undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f16 = fcmp false undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f32 = fcmp false <2 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f32 = fcmp false <4 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f32 = fcmp false <8 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f32 = fcmp false <16 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32 = fcmp false undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32 = fcmp false undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32 = fcmp false undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f32 = fcmp false undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f32 = fcmp false undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f64 = fcmp false <2 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f64 = fcmp false <4 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f64 = fcmp false <8 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64 = fcmp false undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f64 = fcmp false undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f64 = fcmp false undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f64 = fcmp false undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; + %v2f16 = fcmp false <2 x half> undef, undef + %v4f16 = fcmp false <4 x half> undef, undef + %v8f16 = fcmp false <8 x half> undef, undef + %v16f16 = fcmp false <16 x half> undef, undef + + %nxv1f16 = fcmp false undef, undef + %nxv2f16 = fcmp false undef, undef + %nxv4f16 = fcmp false undef, undef + %nxv8f16 = fcmp false undef, undef + %nxv16f16 = fcmp false undef, undef + + %v2f32 = fcmp false <2 x float> undef, undef + %v4f32 = fcmp false <4 x float> undef, undef + %v8f32 = fcmp false <8 x float> undef, undef + %v16f32 = fcmp false <16 x float> undef, undef + + %nxv1f32 = fcmp false undef, undef + %nxv2f32 = fcmp false undef, undef + %nxv4f32 = fcmp false undef, undef + %nxv8f32 = fcmp false undef, undef + %nxv16f32 = fcmp false undef, undef + + %v2f64 = fcmp false <2 x double> undef, undef + %v4f64 = fcmp false <4 x double> undef, undef + %v8f64 = fcmp false <8 x double> undef, undef + + %nxv1f64 = fcmp false undef, undef + %nxv2f64 = fcmp false undef, undef + %nxv4f64 = fcmp false undef, undef + %nxv8f64 = fcmp false undef, undef + + ret void +} diff --git a/llvm/test/Analysis/CostModel/RISCV/rvv-select.ll b/llvm/test/Analysis/CostModel/RISCV/rvv-select.ll index 6dcbc73674aa..b80096b057a6 100644 --- a/llvm/test/Analysis/CostModel/RISCV/rvv-select.ll +++ b/llvm/test/Analysis/CostModel/RISCV/rvv-select.ll @@ -22,14 +22,14 @@ define void @select() { ; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %15 = select i1 undef, undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %16 = select i1 undef, undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %17 = select i1 undef, undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %18 = select i1 undef, undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %19 = select i1 undef, undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %18 = select i1 undef, undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %19 = select i1 undef, undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %20 = select undef, undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %21 = select undef, undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %22 = select undef, undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %23 = select undef, undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %24 = select undef, undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %25 = select undef, undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %24 = select undef, undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %25 = select undef, undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %26 = select i1 undef, i8 undef, i8 undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %27 = select i1 undef, <1 x i8> undef, <1 x i8> undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %28 = select i1 undef, <2 x i8> undef, <2 x i8> undef -- GitLab From 525d00e5edc4b83105c6ad518926b174f76c3a1c Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Thu, 18 Apr 2024 10:56:20 +0900 Subject: [PATCH 115/862] [InstCombine] Fix poison propagation in round up alignment fold We can't directly use the high bits value if it is more poisonous due to poison elements in the masks. This fixes the issue reported in https://github.com/llvm/llvm-project/pull/88217#issuecomment-2061034941. --- .../InstCombine/InstCombineSelect.cpp | 3 ++- .../integer-round-up-pow2-alignment.ll | 20 +++++++++++++------ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp index 2d78fcee1152..41c3ab85940e 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp @@ -2965,7 +2965,8 @@ foldRoundUpIntegerWithPow2Alignment(SelectInst &SI, return nullptr; if (!XBiasedHighBits->hasOneUse()) { - if (*BiasCst == *LowBitMaskCst) + // We can't directly return XBiasedHighBits if it is more poisonous. + if (*BiasCst == *LowBitMaskCst && impliesPoison(XBiasedHighBits, X)) return XBiasedHighBits; return nullptr; } diff --git a/llvm/test/Transforms/InstCombine/integer-round-up-pow2-alignment.ll b/llvm/test/Transforms/InstCombine/integer-round-up-pow2-alignment.ll index c7e0553992b9..afd56abf40a5 100644 --- a/llvm/test/Transforms/InstCombine/integer-round-up-pow2-alignment.ll +++ b/llvm/test/Transforms/InstCombine/integer-round-up-pow2-alignment.ll @@ -437,14 +437,17 @@ define i8 @t17_oneuse(i8 %x) { ret i8 %x.roundedup } -; Bias is equal to the alignment-1 (as opposed to alignment), -; so we can just replace %x.roundedup with %x.biased.highbits +; Negative test: We can't replace with %x.biased.highbits because it is +; more poisonous. define <2 x i4> @t18_replacement_0b0001(<2 x i4> %x) { ; CHECK-LABEL: @t18_replacement_0b0001( -; CHECK-NEXT: [[X_BIASED:%.*]] = add <2 x i4> [[X:%.*]], +; CHECK-NEXT: [[X_LOWBITS:%.*]] = and <2 x i4> [[X:%.*]], +; CHECK-NEXT: [[X_LOWBITS_ARE_ZERO:%.*]] = icmp eq <2 x i4> [[X_LOWBITS]], zeroinitializer +; CHECK-NEXT: [[X_BIASED:%.*]] = add <2 x i4> [[X]], ; CHECK-NEXT: [[X_BIASED_HIGHBITS:%.*]] = and <2 x i4> [[X_BIASED]], ; CHECK-NEXT: call void @use.v2i4(<2 x i4> [[X_BIASED_HIGHBITS]]) -; CHECK-NEXT: ret <2 x i4> [[X_BIASED_HIGHBITS]] +; CHECK-NEXT: [[X_ROUNDEDUP:%.*]] = select <2 x i1> [[X_LOWBITS_ARE_ZERO]], <2 x i4> [[X]], <2 x i4> [[X_BIASED_HIGHBITS]] +; CHECK-NEXT: ret <2 x i4> [[X_ROUNDEDUP]] ; %x.lowbits = and <2 x i4> %x, %x.lowbits.are.zero = icmp eq <2 x i4> %x.lowbits, @@ -454,12 +457,17 @@ define <2 x i4> @t18_replacement_0b0001(<2 x i4> %x) { %x.roundedup = select <2 x i1> %x.lowbits.are.zero, <2 x i4> %x, <2 x i4> %x.biased.highbits ret <2 x i4> %x.roundedup } +; Negative test: We can't replace with %x.biased.highbits because it is +; more poisonous. define <2 x i4> @t18_replacement_0b0010(<2 x i4> %x) { ; CHECK-LABEL: @t18_replacement_0b0010( -; CHECK-NEXT: [[X_BIASED:%.*]] = add <2 x i4> [[X:%.*]], +; CHECK-NEXT: [[X_LOWBITS:%.*]] = and <2 x i4> [[X:%.*]], +; CHECK-NEXT: [[X_LOWBITS_ARE_ZERO:%.*]] = icmp eq <2 x i4> [[X_LOWBITS]], zeroinitializer +; CHECK-NEXT: [[X_BIASED:%.*]] = add <2 x i4> [[X]], ; CHECK-NEXT: [[X_BIASED_HIGHBITS:%.*]] = and <2 x i4> [[X_BIASED]], ; CHECK-NEXT: call void @use.v2i4(<2 x i4> [[X_BIASED_HIGHBITS]]) -; CHECK-NEXT: ret <2 x i4> [[X_BIASED_HIGHBITS]] +; CHECK-NEXT: [[X_ROUNDEDUP:%.*]] = select <2 x i1> [[X_LOWBITS_ARE_ZERO]], <2 x i4> [[X]], <2 x i4> [[X_BIASED_HIGHBITS]] +; CHECK-NEXT: ret <2 x i4> [[X_ROUNDEDUP]] ; %x.lowbits = and <2 x i4> %x, %x.lowbits.are.zero = icmp eq <2 x i4> %x.lowbits, -- GitLab From 0ee260ec37a039ffc276f2f827c842f71513a3e9 Mon Sep 17 00:00:00 2001 From: Kai Luo Date: Thu, 18 Apr 2024 10:00:56 +0800 Subject: [PATCH 116/862] [PowerPC] `ANDI_rec_1_*` should define CR0 (#89034) These pseudo instructions finally copy the result to CR0 so they should define `CR0` in their definitions. --- llvm/lib/Target/PowerPC/PPCInstrInfo.td | 2 ++ 1 file changed, 2 insertions(+) diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/llvm/lib/Target/PowerPC/PPCInstrInfo.td index 43e3902cf407..261b9a3d1dff 100644 --- a/llvm/lib/Target/PowerPC/PPCInstrInfo.td +++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.td @@ -4191,6 +4191,7 @@ def : Pat<(v4i32 (selectcc i1:$lhs, i1:$rhs, v4i32:$tval, v4i32:$fval, SETUGT)), def : Pat<(v4i32 (selectcc i1:$lhs, i1:$rhs, v4i32:$tval, v4i32:$fval, SETNE)), (SELECT_VRRC (CRXOR $lhs, $rhs), $tval, $fval)>; +let Defs = [CR0] in { def ANDI_rec_1_EQ_BIT : PPCCustomInserterPseudo<(outs crbitrc:$dst), (ins gprc:$in), "#ANDI_rec_1_EQ_BIT", [(set i1:$dst, (trunc (not i32:$in)))]>; @@ -4204,6 +4205,7 @@ def ANDI_rec_1_EQ_BIT8 : PPCCustomInserterPseudo<(outs crbitrc:$dst), (ins g8rc: def ANDI_rec_1_GT_BIT8 : PPCCustomInserterPseudo<(outs crbitrc:$dst), (ins g8rc:$in), "#ANDI_rec_1_GT_BIT8", [(set i1:$dst, (trunc i64:$in))]>; +} def : Pat<(i1 (not (trunc i32:$in))), (ANDI_rec_1_EQ_BIT $in)>; -- GitLab From 29ecd6d50f1400e3101f27567b535eb8af905f58 Mon Sep 17 00:00:00 2001 From: Owen Pan Date: Wed, 17 Apr 2024 19:51:46 -0700 Subject: [PATCH 117/862] [clang-format] Revert breaking stream operators to previous default (#89016) Reverts commit d68826dfbd98, which changes the previous default behavior of always breaking before a stream insertion operator `<<` if both operands are string literals. Also reverts the related commits 27f547968cce and bf05be5b87fc. See the discussion in #88483. --- clang/lib/Format/TokenAnnotator.cpp | 8 ++------ clang/unittests/Format/FormatTest.cpp | 7 +------ clang/unittests/Format/TokenAnnotatorTest.cpp | 9 --------- polly/lib/Analysis/DependenceInfo.cpp | 4 ++-- polly/lib/Analysis/ScopBuilder.cpp | 7 ++++--- 5 files changed, 9 insertions(+), 26 deletions(-) diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp index 628f70417866..80e5605fb577 100644 --- a/clang/lib/Format/TokenAnnotator.cpp +++ b/clang/lib/Format/TokenAnnotator.cpp @@ -5595,12 +5595,8 @@ bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line, return true; if (Left.IsUnterminatedLiteral) return true; - // FIXME: Breaking after newlines seems useful in general. Turn this into an - // option and recognize more cases like endl etc, and break independent of - // what comes after operator lessless. - if (Right.is(tok::lessless) && Right.Next && - Right.Next->is(tok::string_literal) && Left.is(tok::string_literal) && - Left.TokenText.ends_with("\\n\"")) { + if (Right.is(tok::lessless) && Right.Next && Left.is(tok::string_literal) && + Right.Next->is(tok::string_literal)) { return true; } if (Right.is(TT_RequiresClause)) { diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp index 4906b3350b5b..bc61b9c089e9 100644 --- a/clang/unittests/Format/FormatTest.cpp +++ b/clang/unittests/Format/FormatTest.cpp @@ -27339,12 +27339,6 @@ TEST_F(FormatTest, PPDirectivesAndCommentsInBracedInit) { getLLVMStyleWithColumns(30)); } -TEST_F(FormatTest, StreamOutputOperator) { - verifyFormat("std::cout << \"foo\" << \"bar\" << baz;"); - verifyFormat("std::cout << \"foo\\n\"\n" - " << \"bar\";"); -} - TEST_F(FormatTest, BreakAdjacentStringLiterals) { constexpr StringRef Code{ "return \"Code\" \"\\0\\52\\26\\55\\55\\0\" \"x013\" \"\\02\\xBA\";"}; @@ -27359,6 +27353,7 @@ TEST_F(FormatTest, BreakAdjacentStringLiterals) { Style.BreakAdjacentStringLiterals = false; verifyFormat(Code, Style); } + } // namespace } // namespace test } // namespace format diff --git a/clang/unittests/Format/TokenAnnotatorTest.cpp b/clang/unittests/Format/TokenAnnotatorTest.cpp index da02ced8c7a9..a71b68064769 100644 --- a/clang/unittests/Format/TokenAnnotatorTest.cpp +++ b/clang/unittests/Format/TokenAnnotatorTest.cpp @@ -2841,15 +2841,6 @@ TEST_F(TokenAnnotatorTest, BraceKind) { EXPECT_BRACE_KIND(Tokens[16], BK_BracedInit); } -TEST_F(TokenAnnotatorTest, StreamOperator) { - auto Tokens = annotate("\"foo\\n\" << aux << \"foo\\n\" << \"foo\";"); - ASSERT_EQ(Tokens.size(), 9u) << Tokens; - EXPECT_FALSE(Tokens[1]->MustBreakBefore); - EXPECT_FALSE(Tokens[3]->MustBreakBefore); - // Only break between string literals if the former ends with \n. - EXPECT_TRUE(Tokens[5]->MustBreakBefore); -} - TEST_F(TokenAnnotatorTest, UnderstandsElaboratedTypeSpecifier) { auto Tokens = annotate("auto foo() -> enum En {}"); ASSERT_EQ(Tokens.size(), 10u) << Tokens; diff --git a/polly/lib/Analysis/DependenceInfo.cpp b/polly/lib/Analysis/DependenceInfo.cpp index 9ee004fbac9e..a530fa7b58fa 100644 --- a/polly/lib/Analysis/DependenceInfo.cpp +++ b/polly/lib/Analysis/DependenceInfo.cpp @@ -951,8 +951,8 @@ public: bool runOnScop(Scop &S) override { DependenceInfo &P = getAnalysis(); - OS << "Printing analysis '" << P.getPassName() << "' for " << "region: '" - << S.getRegion().getNameStr() << "' in function '" + OS << "Printing analysis '" << P.getPassName() << "' for " + << "region: '" << S.getRegion().getNameStr() << "' in function '" << S.getFunction().getName() << "':\n"; P.printScop(OS, S); diff --git a/polly/lib/Analysis/ScopBuilder.cpp b/polly/lib/Analysis/ScopBuilder.cpp index 214e4d360d6b..a34f29315272 100644 --- a/polly/lib/Analysis/ScopBuilder.cpp +++ b/polly/lib/Analysis/ScopBuilder.cpp @@ -2715,9 +2715,10 @@ void ScopBuilder::addUserContext() { if (NameContext != NameUserContext) { std::string SpaceStr = stringFromIslObj(Space, "null"); errs() << "Error: the name of dimension " << i - << " provided in -polly-context " << "is '" << NameUserContext - << "', but the name in the computed " << "context is '" - << NameContext << "'. Due to this name mismatch, " + << " provided in -polly-context " + << "is '" << NameUserContext << "', but the name in the computed " + << "context is '" << NameContext + << "'. Due to this name mismatch, " << "the -polly-context option is ignored. Please provide " << "the context in the parameter space: " << SpaceStr << ".\n"; return; -- GitLab From b6cc667190e3bee7485a225d3dadd8a57c0a22b6 Mon Sep 17 00:00:00 2001 From: Owen Pan Date: Wed, 17 Apr 2024 19:53:24 -0700 Subject: [PATCH 118/862] [clang-format] Annotate ampamp after new/delete as BinaryOperator (#89033) Fixes #78789. --- clang/lib/Format/TokenAnnotator.cpp | 2 ++ clang/unittests/Format/TokenAnnotatorTest.cpp | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp index 80e5605fb577..a679683077ac 100644 --- a/clang/lib/Format/TokenAnnotator.cpp +++ b/clang/lib/Format/TokenAnnotator.cpp @@ -2912,6 +2912,8 @@ private: return TT_UnaryOperator; if (PrevToken->is(TT_TypeName)) return TT_PointerOrReference; + if (PrevToken->isOneOf(tok::kw_new, tok::kw_delete) && Tok.is(tok::ampamp)) + return TT_BinaryOperator; const FormatToken *NextToken = Tok.getNextNonComment(); diff --git a/clang/unittests/Format/TokenAnnotatorTest.cpp b/clang/unittests/Format/TokenAnnotatorTest.cpp index a71b68064769..4f445c64ab30 100644 --- a/clang/unittests/Format/TokenAnnotatorTest.cpp +++ b/clang/unittests/Format/TokenAnnotatorTest.cpp @@ -309,6 +309,16 @@ TEST_F(TokenAnnotatorTest, UnderstandsUsesOfStarAndAmp) { EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_OverloadedOperatorLParen); EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_FunctionLBrace); EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference); + + Tokens = annotate("if (new && num) {\n" + " new = 1;\n" + "}\n" + "if (!delete && num) {\n" + " delete = 1;\n" + "}"); + ASSERT_EQ(Tokens.size(), 26u) << Tokens; + EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_BinaryOperator); + EXPECT_TOKEN(Tokens[16], tok::ampamp, TT_BinaryOperator); } TEST_F(TokenAnnotatorTest, UnderstandsUsesOfPlusAndMinus) { -- GitLab From f2695a1c2f561da42b973187a254b6eeb7da76a9 Mon Sep 17 00:00:00 2001 From: Chuanqi Xu Date: Wed, 17 Apr 2024 16:40:00 +0800 Subject: [PATCH 119/862] [C++20] [Modules] Avoid writing untouched DeclUpdates from GMF in Reduced BMI Mitigate https://github.com/llvm/llvm-project/issues/61447 The root cause of the above problem is that when we write a declaration, we need to lookup all the redeclarations in the imported modules. Then it will be pretty slow if there are too many redeclarations in different modules. This patch doesn't solve the porblem. What the patchs mitigated is, when we writing a named module, we shouldn't write the declarations from GMF if it is unreferenced **in current module unit**. The difference here is that, if the declaration is used in the imported modules, we used to emit it as an update. But we definitely want to avoid that after this patch. For that reproducer in https://github.com/llvm/llvm-project/issues/61447, it used to take 2.5s to compile and now it only takes 0.49s to compile, which is a big win. --- clang/include/clang/AST/ASTMutationListener.h | 27 ++++++ clang/include/clang/AST/Decl.h | 2 +- clang/include/clang/Serialization/ASTWriter.h | 10 ++ clang/lib/AST/ASTContext.cpp | 18 +++- clang/lib/AST/Decl.cpp | 7 ++ clang/lib/Frontend/MultiplexConsumer.cpp | 29 ++++++ clang/lib/Sema/SemaModule.cpp | 4 + clang/lib/Serialization/ASTWriter.cpp | 64 +++++++++---- .../reduced-bmi-empty-module-purview.cppm | 96 +++++++++++++++++++ 9 files changed, 231 insertions(+), 26 deletions(-) create mode 100644 clang/test/Modules/reduced-bmi-empty-module-purview.cppm diff --git a/clang/include/clang/AST/ASTMutationListener.h b/clang/include/clang/AST/ASTMutationListener.h index 8879f9f3229f..2c4ec2ce67f3 100644 --- a/clang/include/clang/AST/ASTMutationListener.h +++ b/clang/include/clang/AST/ASTMutationListener.h @@ -27,6 +27,7 @@ namespace clang { class FunctionTemplateDecl; class Module; class NamedDecl; + class NamespaceDecl; class ObjCCategoryDecl; class ObjCContainerDecl; class ObjCInterfaceDecl; @@ -35,6 +36,7 @@ namespace clang { class QualType; class RecordDecl; class TagDecl; + class TranslationUnitDecl; class ValueDecl; class VarDecl; class VarTemplateDecl; @@ -147,6 +149,31 @@ public: virtual void AddedAttributeToRecord(const Attr *Attr, const RecordDecl *Record) {} + /// The parser find the named module declaration. + virtual void EnteringModulePurview() {} + + /// An mangling number was added to a Decl + /// + /// \param D The decl that got a mangling number + /// + /// \param Number The mangling number that was added to the Decl + virtual void AddedManglingNumber(const Decl *D, unsigned Number) {} + + /// An static local number was added to a Decl + /// + /// \param D The decl that got a static local number + /// + /// \param Number The static local number that was added to the Decl + virtual void AddedStaticLocalNumbers(const Decl *D, unsigned Number) {} + + /// An anonymous namespace was added the translation unit decl + /// + /// \param TU The translation unit decl that got a new anonymous namespace + /// + /// \param AnonNamespace The anonymous namespace that was added + virtual void AddedAnonymousNamespace(const TranslationUnitDecl *TU, + NamespaceDecl *AnonNamespace) {} + // NOTE: If new methods are added they should also be added to // MultiplexASTMutationListener. }; diff --git a/clang/include/clang/AST/Decl.h b/clang/include/clang/AST/Decl.h index 01af50ca694f..91449ebcceec 100644 --- a/clang/include/clang/AST/Decl.h +++ b/clang/include/clang/AST/Decl.h @@ -120,7 +120,7 @@ public: ASTContext &getASTContext() const { return Ctx; } NamespaceDecl *getAnonymousNamespace() const { return AnonymousNamespace; } - void setAnonymousNamespace(NamespaceDecl *D) { AnonymousNamespace = D; } + void setAnonymousNamespace(NamespaceDecl *D); static TranslationUnitDecl *Create(ASTContext &C); diff --git a/clang/include/clang/Serialization/ASTWriter.h b/clang/include/clang/Serialization/ASTWriter.h index 443f77031047..892d7363e06d 100644 --- a/clang/include/clang/Serialization/ASTWriter.h +++ b/clang/include/clang/Serialization/ASTWriter.h @@ -399,6 +399,11 @@ private: /// record containing modifications to them. DeclUpdateMap DeclUpdates; + /// DeclUpdates added during parsing the GMF. We split these from + /// DeclUpdates since we want to add these updates in GMF on need. + /// Only meaningful for reduced BMI. + DeclUpdateMap DeclUpdatesFromGMF; + using FirstLatestDeclMap = llvm::DenseMap; /// Map of first declarations from a chained PCH that point to the @@ -866,6 +871,11 @@ private: void RedefinedHiddenDefinition(const NamedDecl *D, Module *M) override; void AddedAttributeToRecord(const Attr *Attr, const RecordDecl *Record) override; + void EnteringModulePurview() override; + void AddedManglingNumber(const Decl *D, unsigned) override; + void AddedStaticLocalNumbers(const Decl *D, unsigned) override; + void AddedAnonymousNamespace(const TranslationUnitDecl *, + NamespaceDecl *AnonNamespace) override; }; /// AST and semantic-analysis consumer that generates a diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp index 6ce233704a58..bcae6d705b78 100644 --- a/clang/lib/AST/ASTContext.cpp +++ b/clang/lib/AST/ASTContext.cpp @@ -12245,8 +12245,13 @@ QualType ASTContext::getRealTypeForBitwidth(unsigned DestWidth, } void ASTContext::setManglingNumber(const NamedDecl *ND, unsigned Number) { - if (Number > 1) - MangleNumbers[ND] = Number; + if (Number <= 1) + return; + + MangleNumbers[ND] = Number; + + if (Listener) + Listener->AddedManglingNumber(ND, Number); } unsigned ASTContext::getManglingNumber(const NamedDecl *ND, @@ -12265,8 +12270,13 @@ unsigned ASTContext::getManglingNumber(const NamedDecl *ND, } void ASTContext::setStaticLocalNumber(const VarDecl *VD, unsigned Number) { - if (Number > 1) - StaticLocalNumbers[VD] = Number; + if (Number <= 1) + return; + + StaticLocalNumbers[VD] = Number; + + if (Listener) + Listener->AddedStaticLocalNumbers(VD, Number); } unsigned ASTContext::getStaticLocalNumber(const VarDecl *VD) const { diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp index 33b6f8611f21..63388d79bbf2 100644 --- a/clang/lib/AST/Decl.cpp +++ b/clang/lib/AST/Decl.cpp @@ -5274,6 +5274,13 @@ TranslationUnitDecl *TranslationUnitDecl::Create(ASTContext &C) { return new (C, (DeclContext *)nullptr) TranslationUnitDecl(C); } +void TranslationUnitDecl::setAnonymousNamespace(NamespaceDecl *D) { + AnonymousNamespace = D; + + if (ASTMutationListener *Listener = Ctx.getASTMutationListener()) + Listener->AddedAnonymousNamespace(this, D); +} + void PragmaCommentDecl::anchor() {} PragmaCommentDecl *PragmaCommentDecl::Create(const ASTContext &C, diff --git a/clang/lib/Frontend/MultiplexConsumer.cpp b/clang/lib/Frontend/MultiplexConsumer.cpp index 737877329c9c..744ea70cc24d 100644 --- a/clang/lib/Frontend/MultiplexConsumer.cpp +++ b/clang/lib/Frontend/MultiplexConsumer.cpp @@ -20,6 +20,9 @@ using namespace clang; namespace clang { +class NamespaceDecl; +class TranslationUnitDecl; + MultiplexASTDeserializationListener::MultiplexASTDeserializationListener( const std::vector& L) : Listeners(L) { @@ -115,6 +118,11 @@ public: void RedefinedHiddenDefinition(const NamedDecl *D, Module *M) override; void AddedAttributeToRecord(const Attr *Attr, const RecordDecl *Record) override; + void EnteringModulePurview() override; + void AddedManglingNumber(const Decl *D, unsigned) override; + void AddedStaticLocalNumbers(const Decl *D, unsigned) override; + void AddedAnonymousNamespace(const TranslationUnitDecl *, + NamespaceDecl *AnonNamespace) override; private: std::vector Listeners; @@ -238,6 +246,27 @@ void MultiplexASTMutationListener::AddedAttributeToRecord( L->AddedAttributeToRecord(Attr, Record); } +void MultiplexASTMutationListener::EnteringModulePurview() { + for (auto *L : Listeners) + L->EnteringModulePurview(); +} + +void MultiplexASTMutationListener::AddedManglingNumber(const Decl *D, + unsigned Number) { + for (auto *L : Listeners) + L->AddedManglingNumber(D, Number); +} +void MultiplexASTMutationListener::AddedStaticLocalNumbers(const Decl *D, + unsigned Number) { + for (auto *L : Listeners) + L->AddedStaticLocalNumbers(D, Number); +} +void MultiplexASTMutationListener::AddedAnonymousNamespace( + const TranslationUnitDecl *TU, NamespaceDecl *AnonNamespace) { + for (auto *L : Listeners) + L->AddedAnonymousNamespace(TU, AnonNamespace); +} + } // end namespace clang MultiplexConsumer::MultiplexConsumer( diff --git a/clang/lib/Sema/SemaModule.cpp b/clang/lib/Sema/SemaModule.cpp index 2ddf9d70263a..67658c93ed3b 100644 --- a/clang/lib/Sema/SemaModule.cpp +++ b/clang/lib/Sema/SemaModule.cpp @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "clang/AST/ASTConsumer.h" +#include "clang/AST/ASTMutationListener.h" #include "clang/Lex/HeaderSearch.h" #include "clang/Lex/Preprocessor.h" #include "clang/Sema/SemaInternal.h" @@ -475,6 +476,9 @@ Sema::ActOnModuleDecl(SourceLocation StartLoc, SourceLocation ModuleLoc, getASTContext().setCurrentNamedModule(Mod); + if (auto *Listener = getASTMutationListener()) + Listener->EnteringModulePurview(); + // We already potentially made an implicit import (in the case of a module // implementation unit importing its interface). Make this module visible // and return the import decl to be added to the current TU. diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp index b2a078b6d80f..c3cf242391b7 100644 --- a/clang/lib/Serialization/ASTWriter.cpp +++ b/clang/lib/Serialization/ASTWriter.cpp @@ -5026,27 +5026,6 @@ ASTFileSignature ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot, GetDeclRef(D); } - // If the translation unit has an anonymous namespace, and we don't already - // have an update block for it, write it as an update block. - // FIXME: Why do we not do this if there's already an update block? - if (NamespaceDecl *NS = TU->getAnonymousNamespace()) { - ASTWriter::UpdateRecord &Record = DeclUpdates[TU]; - if (Record.empty()) - Record.push_back(DeclUpdate(UPD_CXX_ADDED_ANONYMOUS_NAMESPACE, NS)); - } - - // Add update records for all mangling numbers and static local numbers. - // These aren't really update records, but this is a convenient way of - // tagging this rare extra data onto the declarations. - for (const auto &Number : Context.MangleNumbers) - if (!Number.first->isFromASTFile()) - DeclUpdates[Number.first].push_back(DeclUpdate(UPD_MANGLING_NUMBER, - Number.second)); - for (const auto &Number : Context.StaticLocalNumbers) - if (!Number.first->isFromASTFile()) - DeclUpdates[Number.first].push_back(DeclUpdate(UPD_STATIC_LOCAL_NUMBER, - Number.second)); - // Make sure visible decls, added to DeclContexts previously loaded from // an AST file, are registered for serialization. Likewise for template // specializations added to imported templates. @@ -5315,6 +5294,41 @@ ASTFileSignature ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot, return backpatchSignature(); } +void ASTWriter::EnteringModulePurview() { + // In C++20 named modules, all entities before entering the module purview + // lives in the GMF. + if (GeneratingReducedBMI) + DeclUpdatesFromGMF.swap(DeclUpdates); +} + +// Add update records for all mangling numbers and static local numbers. +// These aren't really update records, but this is a convenient way of +// tagging this rare extra data onto the declarations. +void ASTWriter::AddedManglingNumber(const Decl *D, unsigned Number) { + if (D->isFromASTFile()) + return; + + DeclUpdates[D].push_back(DeclUpdate(UPD_MANGLING_NUMBER, Number)); +} +void ASTWriter::AddedStaticLocalNumbers(const Decl *D, unsigned Number) { + if (D->isFromASTFile()) + return; + + DeclUpdates[D].push_back(DeclUpdate(UPD_STATIC_LOCAL_NUMBER, Number)); +} + +void ASTWriter::AddedAnonymousNamespace(const TranslationUnitDecl *TU, + NamespaceDecl *AnonNamespace) { + // If the translation unit has an anonymous namespace, and we don't already + // have an update block for it, write it as an update block. + // FIXME: Why do we not do this if there's already an update block? + if (NamespaceDecl *NS = TU->getAnonymousNamespace()) { + ASTWriter::UpdateRecord &Record = DeclUpdates[TU]; + if (Record.empty()) + Record.push_back(DeclUpdate(UPD_CXX_ADDED_ANONYMOUS_NAMESPACE, NS)); + } +} + void ASTWriter::WriteDeclAndTypes(ASTContext &Context) { // Keep writing types, declarations, and declaration update records // until we've emitted all of them. @@ -5860,6 +5874,14 @@ DeclID ASTWriter::GetDeclRef(const Decl *D) { return 0; } + // If the DeclUpdate from the GMF gets touched, emit it. + if (auto *Iter = DeclUpdatesFromGMF.find(D); + Iter != DeclUpdatesFromGMF.end()) { + for (DeclUpdate &Update : Iter->second) + DeclUpdates[D].push_back(Update); + DeclUpdatesFromGMF.erase(Iter); + } + // If D comes from an AST file, its declaration ID is already known and // fixed. if (D->isFromASTFile()) diff --git a/clang/test/Modules/reduced-bmi-empty-module-purview.cppm b/clang/test/Modules/reduced-bmi-empty-module-purview.cppm new file mode 100644 index 000000000000..54a1e9b77e6e --- /dev/null +++ b/clang/test/Modules/reduced-bmi-empty-module-purview.cppm @@ -0,0 +1,96 @@ +// Test that we won't write additional information into the Reduced BMI if the +// module purview is empty. +// +// RUN: rm -rf %t +// RUN: mkdir -p %t +// RUN: split-file %s %t +// +// RUN: %clang_cc1 -std=c++20 %t/M.cppm -emit-reduced-module-interface -o %t/M.pcm +// RUN: %clang_cc1 -std=c++20 %t/A.cppm -emit-reduced-module-interface -o %t/A.pcm \ +// RUN: -fmodule-file=M=%t/M.pcm +// RUN: llvm-bcanalyzer --dump --disable-histogram --show-binary-blobs %t/A.pcm > %t/A.dump +// RUN: cat %t/A.dump | FileCheck %t/A.cppm +// +// RUN: %clang_cc1 -std=c++20 %t/A1.cppm -emit-reduced-module-interface -o %t/A1.pcm \ +// RUN: -fmodule-file=M=%t/M.pcm +// RUN: llvm-bcanalyzer --dump --disable-histogram --show-binary-blobs %t/A1.pcm > %t/A1.dump +// RUN: cat %t/A1.dump | FileCheck %t/A1.cppm + +//--- foo.h +namespace ns { +template +class A { + +}; + +extern template class A; + +inline A a() { return A(); } +template +A _av_ = A(); + +auto _av_1 = _av_; +auto _av_2 = _av_; + +template <> +class A { + +}; + +void func(A, ...) { + +} + +} + +struct S { + union { + unsigned int V; + struct { + int v1; + int v2; + ns::A a1; + } WESQ; + }; + + union { + double d; + struct { + int v1; + unsigned v2; + ns::A a1; + } Another; + }; +}; + +//--- M.cppm +module; +#include "foo.h" +export module M; +export namespace nv { + using ns::A; + using ns::a; + using ns::_av_; + + using ns::func; +} +using ::S; + +//--- A.cppm +module; +#include "foo.h" +export module A; +import M; + +// CHECK-NOT: Date: Thu, 18 Apr 2024 11:04:06 +0800 Subject: [PATCH 120/862] [RISCV] Check that VLMAX is the same when demanding exact VL (#89080) --- llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp index fa37d1ccccd7..331253e39c0a 100644 --- a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp +++ b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp @@ -616,7 +616,7 @@ public: if (SEWLMULRatioOnly) return false; - if (Used.VLAny && !hasSameAVL(Require)) + if (Used.VLAny && !(hasSameAVL(Require) && hasSameVLMAX(Require))) return false; if (Used.VLZeroness && !hasEquallyZeroAVL(Require, MRI)) -- GitLab From 3e2aad412ada8f1db88c9e0f984a302deed74f4b Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Wed, 17 Apr 2024 21:03:35 -0700 Subject: [PATCH 121/862] [RISCV] Speed up RISCVRegisterInfo::needsFrameBaseReg when frame pointer isn't used. NFC (#89163) The callee saved size is only used if there is a frame pointer. Sink the code onto the frame pointer only path. --- llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp | 22 +++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp index 84af6eec40ee..367a62e830cb 100644 --- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp @@ -607,20 +607,22 @@ bool RISCVRegisterInfo::needsFrameBaseReg(MachineInstr *MI, const MachineFrameInfo &MFI = MF.getFrameInfo(); const RISCVFrameLowering *TFI = getFrameLowering(MF); const MachineRegisterInfo &MRI = MF.getRegInfo(); - unsigned CalleeSavedSize = 0; Offset += getFrameIndexInstrOffset(MI, FIOperandNum); - // Estimate the stack size used to store callee saved registers( - // excludes reserved registers). - BitVector ReservedRegs = getReservedRegs(MF); - for (const MCPhysReg *R = MRI.getCalleeSavedRegs(); MCPhysReg Reg = *R; ++R) { - if (!ReservedRegs.test(Reg)) - CalleeSavedSize += getSpillSize(*getMinimalPhysRegClass(Reg)); - } + if (TFI->hasFP(MF) && !shouldRealignStack(MF)) { + // Estimate the stack size used to store callee saved registers( + // excludes reserved registers). + unsigned CalleeSavedSize = 0; + BitVector ReservedRegs = getReservedRegs(MF); + for (const MCPhysReg *R = MRI.getCalleeSavedRegs(); MCPhysReg Reg = *R; + ++R) { + if (!ReservedRegs.test(Reg)) + CalleeSavedSize += getSpillSize(*getMinimalPhysRegClass(Reg)); + } - int64_t MaxFPOffset = Offset - CalleeSavedSize; - if (TFI->hasFP(MF) && !shouldRealignStack(MF)) + int64_t MaxFPOffset = Offset - CalleeSavedSize; return !isFrameOffsetLegal(MI, RISCV::X8, MaxFPOffset); + } // Assume 128 bytes spill slots size to estimate the maximum possible // offset relative to the stack pointer. -- GitLab From 8b37ec1f7bda40c240f7bfda6737df5043860103 Mon Sep 17 00:00:00 2001 From: Will Hawkins Date: Thu, 18 Apr 2024 00:29:57 -0400 Subject: [PATCH 122/862] [libc++][NFC] Add additional tests for begin/end of std::ranges::take_view (#79085) Add additional tests for `begin`/`end` of `std::ranges::take_view`. In partial fulfillment of #72406. --- .../range.adaptors/range.take/begin.pass.cpp | 100 +++++++++++++----- .../range.adaptors/range.take/end.pass.cpp | 14 +++ .../ranges/range.adaptors/range.take/types.h | 50 +++++++++ 3 files changed, 139 insertions(+), 25 deletions(-) diff --git a/libcxx/test/std/ranges/range.adaptors/range.take/begin.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.take/begin.pass.cpp index 1873481d7322..9f11a991535e 100644 --- a/libcxx/test/std/ranges/range.adaptors/range.take/begin.pass.cpp +++ b/libcxx/test/std/ranges/range.adaptors/range.take/begin.pass.cpp @@ -11,8 +11,9 @@ // constexpr auto begin() requires (!simple-view); // constexpr auto begin() const requires range; -#include #include +#include +#include #include "test_macros.h" #include "test_iterators.h" @@ -27,55 +28,104 @@ struct NonCommonSimpleView : std::ranges::view_base { static_assert(std::ranges::sized_range); static_assert(!std::ranges::sized_range); +using CommonInputIterPtrConstInt = common_input_iterator; +using CountedCommonInputIterPtrConstInt = std::counted_iterator; + constexpr bool test() { int buffer[8] = {1, 2, 3, 4, 5, 6, 7, 8}; - // sized_range && random_access_iterator + // simple-view && sized_range && random_access_range { - std::ranges::take_view tv(SizedRandomAccessView(buffer), 4); - assert(tv.begin() == SizedRandomAccessView(buffer).begin()); - ASSERT_SAME_TYPE(decltype(tv.begin()), RandomAccessIter); - } + using ViewTested = SizedRandomAccessView; + static_assert(simple_view); + static_assert(std::ranges::sized_range); + static_assert(std::ranges::random_access_range); - { - const std::ranges::take_view tv(SizedRandomAccessView(buffer), 4); - assert(tv.begin() == SizedRandomAccessView(buffer).begin()); + std::ranges::take_view tv(ViewTested(buffer), 4); + assert(tv.begin() == ViewTested(buffer).begin()); ASSERT_SAME_TYPE(decltype(tv.begin()), RandomAccessIter); - } - // sized_range && !random_access_iterator - { - std::ranges::take_view tv(SizedForwardView{buffer}, 4); - assert(tv.begin() == std::counted_iterator(ForwardIter(buffer), 4)); - ASSERT_SAME_TYPE(decltype(tv.begin()), std::counted_iterator); + const std::ranges::take_view ctv(ViewTested(buffer), 4); + assert(ctv.begin() == ViewTested(buffer).begin()); + ASSERT_SAME_TYPE(decltype(ctv.begin()), RandomAccessIter); } + // simple-view && sized_range && !random_access_range { - const std::ranges::take_view tv(SizedForwardView{buffer}, 4); - assert(tv.begin() == std::counted_iterator(ForwardIter(buffer), 4)); + using ViewTested = SizedForwardView; + static_assert(simple_view); + static_assert(std::ranges::sized_range); + static_assert(!std::ranges::random_access_range); + + std::ranges::take_view tv(ViewTested{buffer}, 16); // underlying size is 8 + assert(tv.begin() == std::counted_iterator(ForwardIter(buffer), 8)); // expect min(8, 16) ASSERT_SAME_TYPE(decltype(tv.begin()), std::counted_iterator); + + const std::ranges::take_view ctv(ViewTested{buffer}, 4); + assert(ctv.begin() == std::counted_iterator(ForwardIter(buffer), 4)); + ASSERT_SAME_TYPE(decltype(ctv.begin()), std::counted_iterator); } - // !sized_range + // simple-view && !sized_range { - std::ranges::take_view tv(MoveOnlyView{buffer}, 4); + using ViewTested = MoveOnlyView; + static_assert(simple_view); + std::ranges::take_view tv(ViewTested{buffer}, 4); assert(tv.begin() == std::counted_iterator(buffer, 4)); ASSERT_SAME_TYPE(decltype(tv.begin()), std::counted_iterator); + + const std::ranges::take_view ctv(ViewTested{buffer}, 4); + assert(ctv.begin() == std::counted_iterator(buffer, 4)); + ASSERT_SAME_TYPE(decltype(ctv.begin()), std::counted_iterator); } + // simple-view && sized_range && !sized_range { - const std::ranges::take_view tv(MoveOnlyView{buffer}, 4); - assert(tv.begin() == std::counted_iterator(buffer, 4)); + using ViewTested = NonCommonSimpleView; + static_assert(simple_view); + static_assert(std::ranges::sized_range); + static_assert(!std::ranges::sized_range); + + std::ranges::take_view tv{}; ASSERT_SAME_TYPE(decltype(tv.begin()), std::counted_iterator); + ASSERT_SAME_TYPE(decltype(std::as_const(tv).begin()), std::counted_iterator); } - // simple-view && sized_range && !size_range + // !simple-view && !sized_range { - std::ranges::take_view tv{}; - ASSERT_SAME_TYPE(decltype(tv.begin()), std::counted_iterator); - ASSERT_SAME_TYPE(decltype(std::as_const(tv).begin()), std::counted_iterator); + using ViewTested = NonSimpleNonSizedView; + static_assert(!simple_view); + static_assert(!std::ranges::sized_range); + + std::ranges::take_view tv{ViewTested{buffer, buffer + 2}, 4}; + // The count for the counted iterator is the count of the take_view (i.e., 4) + assert(tv.begin() == CountedCommonInputIterPtrConstInt(CommonInputIterPtrConstInt(buffer), 4)); + ASSERT_SAME_TYPE(decltype(tv.begin()), CountedCommonInputIterPtrConstInt); } + // !simple-view && sized_range + { + using ViewTested = NonSimpleSizedView; + static_assert(!simple_view); + static_assert(std::ranges::sized_range); + + std::ranges::take_view tv{ViewTested{buffer, buffer + 2}, 4}; + // The count for the counted iterator is the min(2, 4) (i.e., 2). + assert(tv.begin() == CountedCommonInputIterPtrConstInt(CommonInputIterPtrConstInt(buffer), 2)); + ASSERT_SAME_TYPE(decltype(tv.begin()), CountedCommonInputIterPtrConstInt); + } + + // !simple-view && sized_range && random_access_range + { + using ViewTested = NonSimpleSizedRandomView; + static_assert(!simple_view); + static_assert(std::ranges::sized_range); + static_assert(std::ranges::random_access_range); + + std::ranges::take_view tv{ViewTested{buffer, buffer + 2}, 4}; + assert(tv.begin() == random_access_iterator(buffer)); + ASSERT_SAME_TYPE(decltype(tv.begin()), random_access_iterator); + } return true; } diff --git a/libcxx/test/std/ranges/range.adaptors/range.take/end.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.take/end.pass.cpp index 6cab05daa9e0..eddb39af2712 100644 --- a/libcxx/test/std/ranges/range.adaptors/range.take/end.pass.cpp +++ b/libcxx/test/std/ranges/range.adaptors/range.take/end.pass.cpp @@ -69,6 +69,20 @@ constexpr bool test() { assert(tv.end() == std::ranges::next(tv.begin(), 8)); } + { + // __iterator has base with type std::ranges::sentinel_t; adding a const qualifier + // would change the equality. + std::ranges::take_view tvns(NonSimpleNonSizedView{buffer, buffer + 8}, 0); + static_assert(!std::is_same_v>); + } + + { + // __iterator has base with type std::ranges::sentinel_t; adding a const qualifier + // would not change the equality. + std::ranges::take_view tvs(SimpleViewNonSized{buffer, buffer + 8}, 0); + static_assert(std::is_same_v>); + } + return true; } diff --git a/libcxx/test/std/ranges/range.adaptors/range.take/types.h b/libcxx/test/std/ranges/range.adaptors/range.take/types.h index db80e68bb21a..7590ce33bffc 100644 --- a/libcxx/test/std/ranges/range.adaptors/range.take/types.h +++ b/libcxx/test/std/ranges/range.adaptors/range.take/types.h @@ -65,4 +65,54 @@ private: int* end_; }; +template